diff --git a/build.xml b/build.xml index 275cb5555..1f26e7b7a 100644 --- a/build.xml +++ b/build.xml @@ -163,6 +163,14 @@ + + + + @@ -709,53 +717,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -769,20 +730,113 @@ - - - + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -792,10 +846,10 @@ - + - - + - - + + + + - - + + + + + + + + + + + + + + - - + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + - + + + - + - + + + - + - + + + - + - + + + - + + + + + + - - + + - - + + - - - - - - + + diff --git a/ivy.xml b/ivy.xml index 115f4062a..f90b9a010 100644 --- a/ivy.xml +++ b/ivy.xml @@ -15,10 +15,8 @@ - - - - + + diff --git a/public/R/queueJobReport.R b/public/R/queueJobReport.R index a24d269c9..de6aa045e 100644 --- a/public/R/queueJobReport.R +++ b/public/R/queueJobReport.R @@ -12,14 +12,14 @@ if ( onCMDLine ) { inputFileName = args[1] outputPDF = args[2] } else { - #inputFileName = "~/Desktop/broadLocal/GATK/unstable/report.txt" - inputFileName = "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/Q-25718@node1149.jobreport.txt" + inputFileName = "~/Desktop/Q-30033@gsa1.jobreport.txt" + #inputFileName = "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/Q-25718@node1149.jobreport.txt" #inputFileName = "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/rodPerformanceGoals/history/report.082711.txt" outputPDF = NA } -RUNTIME_UNITS = "(sec)" -ORIGINAL_UNITS_TO_SECONDS = 1/1000 +RUNTIME_UNITS = "(hours)" +ORIGINAL_UNITS_TO_SECONDS = 1/1000/60/60 # # Helper function to aggregate all of the jobs in the report across all tables @@ -33,7 +33,7 @@ allJobsFromReport <- function(report) { # # Creates segmentation plots of time (x) vs. job (y) with segments for the duration of the job # -plotJobsGantt <- function(gatkReport, sortOverall) { +plotJobsGantt <- function(gatkReport, sortOverall, includeText) { allJobs = allJobsFromReport(gatkReport) if ( sortOverall ) { title = "All jobs, by analysis, by start time" @@ -44,16 +44,18 @@ plotJobsGantt <- function(gatkReport, sortOverall) { } allJobs$index = 1:nrow(allJobs) minTime = min(allJobs$startTime) - allJobs$relStartTime = allJobs$startTime - minTime - allJobs$relDoneTime = allJobs$doneTime - minTime + allJobs$relStartTime = (allJobs$startTime - minTime) * ORIGINAL_UNITS_TO_SECONDS + allJobs$relDoneTime = (allJobs$doneTime - minTime) * ORIGINAL_UNITS_TO_SECONDS allJobs$ganttName = paste(allJobs$jobName, "@", allJobs$exechosts) maxRelTime = max(allJobs$relDoneTime) p <- ggplot(data=allJobs, aes(x=relStartTime, y=index, color=analysisName)) - p <- p + geom_segment(aes(xend=relDoneTime, yend=index), size=2, arrow=arrow(length = unit(0.1, "cm"))) - p <- p + geom_text(aes(x=relDoneTime, label=ganttName, hjust=-0.2), size=2) + p <- p + theme_bw() + p <- p + geom_segment(aes(xend=relDoneTime, yend=index), size=1, arrow=arrow(length = unit(0.1, "cm"))) + if ( includeText ) + p <- p + geom_text(aes(x=relDoneTime, label=ganttName, hjust=-0.2), size=2) p <- p + xlim(0, maxRelTime * 1.1) p <- p + xlab(paste("Start time (relative to first job)", RUNTIME_UNITS)) - p <- p + ylab("Job") + p <- p + ylab("Job number") p <- p + opts(title=title) print(p) } @@ -140,6 +142,8 @@ print(paste("Project :", inputFileName)) convertUnits <- function(gatkReportData) { convertGroup <- function(g) { g$runtime = g$runtime * ORIGINAL_UNITS_TO_SECONDS + g$startTime = g$startTime * ORIGINAL_UNITS_TO_SECONDS + g$doneTime = g$doneTime * ORIGINAL_UNITS_TO_SECONDS g } lapply(gatkReportData, convertGroup) @@ -155,8 +159,8 @@ if ( ! is.na(outputPDF) ) { pdf(outputPDF, height=8.5, width=11) } -plotJobsGantt(gatkReportData, T) -plotJobsGantt(gatkReportData, F) +plotJobsGantt(gatkReportData, T, F) +plotJobsGantt(gatkReportData, F, F) plotProgressByTime(gatkReportData) for ( group in gatkReportData ) { plotGroup(group) diff --git a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java index 7ea515591..1ef452a5c 100755 --- a/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java +++ b/public/java/src/org/broadinstitute/sting/analyzecovariates/AnalyzeCovariates.java @@ -114,7 +114,7 @@ public class AnalyzeCovariates extends CommandLineProgram { private String RECAL_FILE = "output.recal_data.csv"; @Argument(fullName = "output_dir", shortName = "outputDir", doc = "The directory in which to output all the plots and intermediate data files", required = false) private String OUTPUT_DIR = "analyzeCovariates/"; - @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/tools/apps/R-2.6.0/bin/Rscript", required = false) + @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false) private String PATH_TO_RSCRIPT = "Rscript"; @Argument(fullName = "path_to_resources", shortName = "resources", doc = "Path to resources folder holding the Sting R scripts.", required = false) private String PATH_TO_RESOURCES = "public/R/"; diff --git a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java index 16358d05f..5fff8f609 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ArgumentTypeDescriptor.java @@ -379,7 +379,7 @@ class RodBindingArgumentTypeDescriptor extends ArgumentTypeDescriptor { } if ( tribbleType == null ) - if ( ! file.canRead() | !! file.isFile() ) { + if ( ! file.canRead() | ! file.isFile() ) { throw new UserException.BadArgumentValue(name, "Couldn't read file to determine type: " + file); } else { throw new UserException.CommandLineException( diff --git a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java index 5b9ebd99b..972943e26 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/GenomeAnalysisEngine.java @@ -929,6 +929,14 @@ public class GenomeAnalysisEngine { return readsDataSource.getHeader(reader); } + /** + * Gets the master sequence dictionary for this GATK engine instance + * @return a never-null dictionary listing all of the contigs known to this engine instance + */ + public SAMSequenceDictionary getMasterSequenceDictionary() { + return getReferenceDataSource().getReference().getSequenceDictionary(); + } + /** * Returns data source object encapsulating all essential info and handlers used to traverse * reads; header merger, individual file readers etc can be accessed through the returned data source object. diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java index 467aebac5..47eb55b28 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/BAMScheduler.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.datasources.reads; import net.sf.picard.util.PeekableIterator; import net.sf.samtools.GATKBAMFileSpan; +import net.sf.samtools.GATKChunk; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocSortedSet; @@ -84,7 +85,7 @@ public class BAMScheduler implements Iterator { if(currentLocus == GenomeLoc.UNMAPPED) { nextFilePointer = new FilePointer(GenomeLoc.UNMAPPED); for(SAMReaderID id: dataSource.getReaderIDs()) - nextFilePointer.addFileSpans(id,new GATKBAMFileSpan()); + nextFilePointer.addFileSpans(id,new GATKBAMFileSpan(new GATKChunk(indexFiles.get(id).getStartOfLastLinearBin(),Long.MAX_VALUE))); currentLocus = null; continue; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java index 5d0c38b78..dc703ff23 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/GATKBAMIndex.java @@ -215,6 +215,45 @@ public class GATKBAMIndex { return (new GATKBin(bin).getBinNumber()-levelStart+1)*(BIN_GENOMIC_SPAN /levelSize); } + /** + * Use to get close to the unmapped reads at the end of a BAM file. + * @return The file offset of the first record in the last linear bin, or -1 + * if there are no elements in linear bins (i.e. no mapped reads). + */ + public long getStartOfLastLinearBin() { + openIndexFile(); + + seek(4); + + final int sequenceCount = readInteger(); + // Because no reads may align to the last sequence in the sequence dictionary, + // grab the last element of the linear index for each sequence, and return + // the last one from the last sequence that has one. + long lastLinearIndexPointer = -1; + for (int i = 0; i < sequenceCount; i++) { + // System.out.println("# Sequence TID: " + i); + final int nBins = readInteger(); + // System.out.println("# nBins: " + nBins); + for (int j1 = 0; j1 < nBins; j1++) { + // Skip bin # + skipBytes(4); + final int nChunks = readInteger(); + // Skip chunks + skipBytes(16 * nChunks); + } + final int nLinearBins = readInteger(); + if (nLinearBins > 0) { + // Skip to last element of list of linear bins + skipBytes(8 * (nLinearBins - 1)); + lastLinearIndexPointer = readLongs(1)[0]; + } + } + + closeIndexFile(); + + return lastLinearIndexPointer; + } + /** * Gets the possible number of bins for a given reference sequence. * @return How many bins could possibly be used according to this indexing scheme to index a single contig. diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/LowMemoryIntervalSharder.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/LowMemoryIntervalSharder.java index ba6321121..bf5f33dc3 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/LowMemoryIntervalSharder.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/LowMemoryIntervalSharder.java @@ -59,7 +59,7 @@ public class LowMemoryIntervalSharder implements Iterator { */ public FilePointer next() { FilePointer current = wrappedIterator.next(); - while(wrappedIterator.hasNext() && current.minus(wrappedIterator.peek()) == 0) + while(wrappedIterator.hasNext() && current.isRegionUnmapped == wrappedIterator.peek().isRegionUnmapped && current.minus(wrappedIterator.peek()) == 0) current = current.combine(parser,wrappedIterator.next()); return current; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShardStrategy.java b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShardStrategy.java index c2235ec73..5ea75dbb0 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShardStrategy.java +++ b/public/java/src/org/broadinstitute/sting/gatk/datasources/reads/ReadShardStrategy.java @@ -134,24 +134,11 @@ public class ReadShardStrategy implements ShardStrategy { Map selectedReaders = new HashMap(); while(selectedReaders.size() == 0 && currentFilePointer != null) { shardPosition = currentFilePointer.fileSpans; + for(SAMReaderID id: shardPosition.keySet()) { - // If the region contains location information (in other words, it is not at - // the start of the unmapped region), add the region. - if(currentFilePointer.isRegionUnmapped) { - // If the region is unmapped and no location data exists, add a null as an indicator to - // start at the next unmapped region. - if(!isIntoUnmappedRegion) { - selectedReaders.put(id,null); - isIntoUnmappedRegion = true; - } - else - selectedReaders.put(id,position.get(id)); - } - else { - SAMFileSpan fileSpan = shardPosition.get(id).removeContentsBefore(position.get(id)); - if(!fileSpan.isEmpty()) - selectedReaders.put(id,fileSpan); - } + SAMFileSpan fileSpan = shardPosition.get(id).removeContentsBefore(position.get(id)); + if(!fileSpan.isEmpty()) + selectedReaders.put(id,fileSpan); } if(selectedReaders.size() > 0) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/examples/GATKDocsExample.java b/public/java/src/org/broadinstitute/sting/gatk/examples/GATKDocsExample.java index 4541a0537..db4f477c2 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/examples/GATKDocsExample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/examples/GATKDocsExample.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.examples; import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.ArgumentCollection; +import org.broadinstitute.sting.commandline.Hidden; import org.broadinstitute.sting.gatk.arguments.StandardVariantContextInputArgumentCollection; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -59,6 +60,7 @@ import org.broadinstitute.sting.gatk.walkers.RodWalker; * @author Your Name * @since Date created */ +@Hidden public class GATKDocsExample extends RodWalker { /** * Put detailed documentation about the argument here. No need to duplicate the summary information diff --git a/public/java/src/org/broadinstitute/sting/gatk/filters/PlatformFilter.java b/public/java/src/org/broadinstitute/sting/gatk/filters/PlatformFilter.java index 30b2f828d..8e241bb2c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/filters/PlatformFilter.java +++ b/public/java/src/org/broadinstitute/sting/gatk/filters/PlatformFilter.java @@ -36,7 +36,7 @@ import org.broadinstitute.sting.utils.sam.ReadUtils; * @version 0.1 */ public class PlatformFilter extends ReadFilter { - @Argument(fullName = "PLFilterName", shortName = "PLFilterName", doc="Discard reads with RG:PL attribute containing this strign", required=false) + @Argument(fullName = "PLFilterName", shortName = "PLFilterName", doc="Discard reads with RG:PL attribute containing this string", required=false) protected String[] PLFilterNames; public boolean filterOut(SAMRecord rec) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java index ebb4cbe66..4ca7b935f 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/storage/VCFWriterStorage.java @@ -46,7 +46,7 @@ public class VCFWriterStorage implements Storage, VCFWriter { else if ( stub.getOutputStream() != null ) { this.file = null; this.stream = stub.getOutputStream(); - writer = new StandardVCFWriter(stream, stub.doNotWriteGenotypes()); + writer = new StandardVCFWriter(stream, stub.getMasterSequenceDictionary(), stub.doNotWriteGenotypes()); } else throw new ReviewedStingException("Unable to create target to which to write; storage was provided with neither a file nor a stream."); @@ -71,7 +71,7 @@ public class VCFWriterStorage implements Storage, VCFWriter { } // The GATK/Tribble can't currently index block-compressed files on the fly. Disable OTF indexing even if the user explicitly asked for it. - return new StandardVCFWriter(file, this.stream, indexOnTheFly && !stub.isCompressed(), stub.doNotWriteGenotypes()); + return new StandardVCFWriter(file, this.stream, stub.getMasterSequenceDictionary(), indexOnTheFly && !stub.isCompressed(), stub.doNotWriteGenotypes()); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java index 936243f9d..82cb43634 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java +++ b/public/java/src/org/broadinstitute/sting/gatk/io/stubs/VCFWriterStub.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.io.stubs; +import net.sf.samtools.SAMSequenceDictionary; import net.sf.samtools.SAMSequenceRecord; import org.broadinstitute.sting.gatk.CommandLineExecutable; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; @@ -150,6 +151,15 @@ public class VCFWriterStub implements Stub, VCFWriter { return isCompressed; } + /** + * Gets the master sequence dictionary from the engine associated with this stub + * @link GenomeAnalysisEngine.getMasterSequenceDictionary + * @return + */ + public SAMSequenceDictionary getMasterSequenceDictionary() { + return engine.getMasterSequenceDictionary(); + } + /** * Should we tell the VCF writer not to write genotypes? * @return true if the writer should not write genotypes. diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index 4d94130a8..5a7658031 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -293,15 +293,16 @@ public class GATKRunReport { * That is, postReport() is guarenteed not to fail for any reason. */ private File postReportToLocalDisk(File rootDir) { + String filename = getID() + ".report.xml.gz"; + File file = new File(rootDir, filename); try { - String filename = getID() + ".report.xml.gz"; - File file = new File(rootDir, filename); postReportToFile(file); logger.debug("Wrote report to " + file); return file; } catch ( Exception e ) { // we catch everything, and no matter what eat the error exceptDuringRunReport("Couldn't read report file", e); + file.delete(); return null; } } @@ -312,6 +313,7 @@ public class GATKRunReport { File localFile = postReportToLocalDisk(new File("./")); logger.debug("Generating GATK report to AWS S3 based on local file " + localFile); if ( localFile != null ) { // we succeeded in creating the local file + localFile.deleteOnExit(); try { // stop us from printing the annoying, and meaningless, mime types warning Logger mimeTypeLogger = Logger.getLogger(org.jets3t.service.utils.Mimetypes.class); @@ -336,14 +338,13 @@ public class GATKRunReport { //logger.info("Uploading " + localFile + " to AWS bucket"); S3Object s3Object = s3Service.putObject(REPORT_BUCKET_NAME, fileObject); logger.debug("Uploaded to AWS: " + s3Object); + logger.info("Uploaded run statistics report to AWS S3"); } catch ( S3ServiceException e ) { exceptDuringRunReport("S3 exception occurred", e); } catch ( NoSuchAlgorithmException e ) { exceptDuringRunReport("Couldn't calculate MD5", e); } catch ( IOException e ) { exceptDuringRunReport("Couldn't read report file", e); - } finally { - localFile.delete(); } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/indexer/RMDIndexer.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/indexer/RMDIndexer.java index 029800aea..9e5a95d10 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/indexer/RMDIndexer.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/indexer/RMDIndexer.java @@ -101,7 +101,7 @@ public class RMDIndexer extends CommandLineProgram { Index index = IndexFactory.createIndex(inputFileSource, codec, approach); // add writing of the sequence dictionary, if supplied - builder.setIndexSequenceDictionary(inputFileSource, index, ref.getSequenceDictionary(), indexFile, false); + builder.validateAndUpdateIndexSequenceDictionary(inputFileSource, index, ref.getSequenceDictionary()); // create the output stream, and write the index LittleEndianOutputStream stream = new LittleEndianOutputStream(new FileOutputStream(indexFile)); diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/IndexDictionaryUtils.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/IndexDictionaryUtils.java new file mode 100644 index 000000000..d133439dc --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/IndexDictionaryUtils.java @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.refdata.tracks; + +import net.sf.samtools.SAMSequenceDictionary; +import net.sf.samtools.SAMSequenceRecord; +import org.apache.log4j.Logger; +import org.broad.tribble.index.Index; +import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; +import org.broadinstitute.sting.utils.SequenceDictionaryUtils; + +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +/** + * Utilities for working with Sequence Dictionaries embedded in tribble indices + * + * @author Your Name + * @since Date created + */ +public class IndexDictionaryUtils { + private final static Logger logger = Logger.getLogger(IndexDictionaryUtils.class); + + // a constant we use for marking sequence dictionary entries in the Tribble index property list + public static final String SequenceDictionaryPropertyPredicate = "DICT:"; + + /** + * get the sequence dictionary from the track, if available. If not, make it from the contig list that is always in the index + * @param index the index file to use + * @return a SAMSequenceDictionary if available, null if unavailable + */ + public static SAMSequenceDictionary getSequenceDictionaryFromProperties(Index index) { + SAMSequenceDictionary dict = new SAMSequenceDictionary(); + for (Map.Entry entry : index.getProperties().entrySet()) { + if (entry.getKey().startsWith(SequenceDictionaryPropertyPredicate)) + dict.addSequence(new SAMSequenceRecord(entry.getKey().substring(SequenceDictionaryPropertyPredicate.length() , entry.getKey().length()), + Integer.valueOf(entry.getValue()))); + } + return dict; + } + + /** + * create the sequence dictionary with the contig list; a backup approach + * @param index the index file to use + * @param dict the sequence dictionary to add contigs to + * @return the filled-in sequence dictionary + */ + static SAMSequenceDictionary createSequenceDictionaryFromContigList(Index index, SAMSequenceDictionary dict) { + LinkedHashSet seqNames = index.getSequenceNames(); + if (seqNames == null) { + return dict; + } + for (String name : seqNames) { + SAMSequenceRecord seq = new SAMSequenceRecord(name, 0); + dict.addSequence(seq); + } + return dict; + } + + public static void setIndexSequenceDictionary(Index index, SAMSequenceDictionary dict) { + for ( SAMSequenceRecord seq : dict.getSequences() ) { + final String contig = IndexDictionaryUtils.SequenceDictionaryPropertyPredicate + seq.getSequenceName(); + final String length = String.valueOf(seq.getSequenceLength()); + index.addProperty(contig,length); + } + } + + public static void validateTrackSequenceDictionary(final String trackName, + final SAMSequenceDictionary trackDict, + final SAMSequenceDictionary referenceDict, + final ValidationExclusion.TYPE validationExclusionType ) { + // if the sequence dictionary is empty (as well as null which means it doesn't have a dictionary), skip validation + if (trackDict == null || trackDict.size() == 0) + logger.info("Track " + trackName + " doesn't have a sequence dictionary built in, skipping dictionary validation"); + else { + Set trackSequences = new TreeSet(); + for (SAMSequenceRecord dictionaryEntry : trackDict.getSequences()) + trackSequences.add(dictionaryEntry.getSequenceName()); + SequenceDictionaryUtils.validateDictionaries(logger, validationExclusionType, trackName, trackDict, "reference", referenceDict); + } + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java index 06d05912a..3b4558579 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilder.java @@ -25,7 +25,6 @@ package org.broadinstitute.sting.gatk.refdata.tracks; import net.sf.samtools.SAMSequenceDictionary; -import net.sf.samtools.SAMSequenceRecord; import org.apache.log4j.Logger; import org.broad.tribble.FeatureCodec; import org.broad.tribble.FeatureSource; @@ -41,7 +40,6 @@ import org.broadinstitute.sting.gatk.arguments.ValidationExclusion; import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet; import org.broadinstitute.sting.gatk.refdata.utils.RMDTriplet.RMDStorageType; import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.SequenceDictionaryUtils; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -52,16 +50,11 @@ import org.broadinstitute.sting.utils.instrumentation.Sizeof; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.util.LinkedHashSet; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - /** - * - * @author aaron + * + * @author aaron * ` * Class RMDTrackBuilder * @@ -76,9 +69,6 @@ public class RMDTrackBuilder { // extends PluginManager { private final static Logger logger = Logger.getLogger(RMDTrackBuilder.class); public final static boolean MEASURE_TRIBBLE_QUERY_PERFORMANCE = false; - // a constant we use for marking sequence dictionary entries in the Tribble index property list - public static final String SequenceDictionaryPropertyPredicate = "DICT:"; - // private sequence dictionary we use to set our tracks with private SAMSequenceDictionary dict = null; @@ -210,13 +200,19 @@ public class RMDTrackBuilder { // extends PluginManager { try { logger.info(String.format(" Index for %s has size in bytes %d", inputFile, Sizeof.getObjectGraphSize(index))); } catch (ReviewedStingException e) { } - sequenceDictionary = getSequenceDictionaryFromProperties(index); + sequenceDictionary = IndexDictionaryUtils.getSequenceDictionaryFromProperties(index); // if we don't have a dictionary in the Tribble file, and we've set a dictionary for this builder, set it in the file if they match if (sequenceDictionary.size() == 0 && dict != null) { File indexFile = Tribble.indexFile(inputFile); - setIndexSequenceDictionary(inputFile,index,dict,indexFile,true); - sequenceDictionary = getSequenceDictionaryFromProperties(index); + validateAndUpdateIndexSequenceDictionary(inputFile, index, dict); + try { // re-write the index + writeIndexToDisk(index,indexFile,new FSLockWithShared(indexFile)); + } catch (IOException e) { + logger.warn("Unable to update index with the sequence dictionary for file " + indexFile + "; this will not effect your run of the GATK"); + } + + sequenceDictionary = IndexDictionaryUtils.getSequenceDictionaryFromProperties(index); } if ( MEASURE_TRIBBLE_QUERY_PERFORMANCE ) @@ -363,88 +359,31 @@ public class RMDTrackBuilder { // extends PluginManager { // this can take a while, let them know what we're doing logger.info("Creating Tribble index in memory for file " + inputFile); Index idx = IndexFactory.createIndex(inputFile, codec, IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME); - setIndexSequenceDictionary(inputFile, idx, dict, null, false); + validateAndUpdateIndexSequenceDictionary(inputFile, idx, dict); return idx; } - - // --------------------------------------------------------------------------------------------------------- - // static functions to work with the sequence dictionaries of indexes - // --------------------------------------------------------------------------------------------------------- - - /** - * get the sequence dictionary from the track, if available. If not, make it from the contig list that is always in the index - * @param index the index file to use - * @return a SAMSequenceDictionary if available, null if unavailable - */ - public static SAMSequenceDictionary getSequenceDictionaryFromProperties(Index index) { - SAMSequenceDictionary dict = new SAMSequenceDictionary(); - for (Map.Entry entry : index.getProperties().entrySet()) { - if (entry.getKey().startsWith(SequenceDictionaryPropertyPredicate)) - dict.addSequence(new SAMSequenceRecord(entry.getKey().substring(SequenceDictionaryPropertyPredicate.length() , entry.getKey().length()), - Integer.valueOf(entry.getValue()))); - } - return dict; - } - - /** - * create the sequence dictionary with the contig list; a backup approach - * @param index the index file to use - * @param dict the sequence dictionary to add contigs to - * @return the filled-in sequence dictionary - */ - private static SAMSequenceDictionary createSequenceDictionaryFromContigList(Index index, SAMSequenceDictionary dict) { - LinkedHashSet seqNames = index.getSequenceNames(); - if (seqNames == null) { - return dict; - } - for (String name : seqNames) { - SAMSequenceRecord seq = new SAMSequenceRecord(name, 0); - dict.addSequence(seq); - } - return dict; - } - /** * set the sequence dictionary of the track. This function checks that the contig listing of the underlying file is compatible. * (that each contig in the index is in the sequence dictionary). * @param inputFile for proper error message formatting. * @param dict the sequence dictionary * @param index the index file - * @param indexFile the index file - * @param rewriteIndex should we rewrite the index when we're done? - * */ - public void setIndexSequenceDictionary(File inputFile, Index index, SAMSequenceDictionary dict, File indexFile, boolean rewriteIndex) { - if (dict == null) return; - - SAMSequenceDictionary currentDict = createSequenceDictionaryFromContigList(index, new SAMSequenceDictionary()); - validateTrackSequenceDictionary(inputFile.getAbsolutePath(),currentDict,dict); + public void validateAndUpdateIndexSequenceDictionary(final File inputFile, final Index index, final SAMSequenceDictionary dict) { + if (dict == null) throw new ReviewedStingException("BUG: dict cannot be null"); // check that every contig in the RMD contig list is at least in the sequence dictionary we're being asked to set - for (SAMSequenceRecord seq : currentDict.getSequences()) { - if (dict.getSequence(seq.getSequenceName()) == null) - continue; - index.addProperty(SequenceDictionaryPropertyPredicate + dict.getSequence(seq.getSequenceName()).getSequenceName(), String.valueOf(dict.getSequence(seq.getSequenceName()).getSequenceLength())); - } - // re-write the index - if (rewriteIndex) try { - writeIndexToDisk(index,indexFile,new FSLockWithShared(indexFile)); - } catch (IOException e) { - logger.warn("Unable to update index with the sequence dictionary for file " + indexFile + "; this will not effect your run of the GATK"); - } + final SAMSequenceDictionary currentDict = IndexDictionaryUtils.createSequenceDictionaryFromContigList(index, new SAMSequenceDictionary()); + validateTrackSequenceDictionary(inputFile.getAbsolutePath(), currentDict, dict); + + // actually update the dictionary in the index + IndexDictionaryUtils.setIndexSequenceDictionary(index, dict); } - - public void validateTrackSequenceDictionary(String trackName, SAMSequenceDictionary trackDict, SAMSequenceDictionary referenceDict) { - // if the sequence dictionary is empty (as well as null which means it doesn't have a dictionary), skip validation - if (trackDict == null || trackDict.size() == 0) - logger.info("Track " + trackName + " doesn't have a sequence dictionary built in, skipping dictionary validation"); - else { - Set trackSequences = new TreeSet(); - for (SAMSequenceRecord dictionaryEntry : trackDict.getSequences()) - trackSequences.add(dictionaryEntry.getSequenceName()); - SequenceDictionaryUtils.validateDictionaries(logger, validationExclusionType, trackName, trackDict, "reference", referenceDict); - } + public void validateTrackSequenceDictionary(final String trackName, + final SAMSequenceDictionary trackDict, + final SAMSequenceDictionary referenceDict ) { + IndexDictionaryUtils.validateTrackSequenceDictionary(trackName, trackDict, referenceDict, validationExclusionType); } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumns.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumns.java old mode 100644 new mode 100755 index a33631c85..a73123b6c --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumns.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumns.java @@ -24,12 +24,14 @@ package org.broadinstitute.sting.gatk.report; +import org.broadinstitute.sting.utils.collections.Pair; + import java.util.*; /** * Tracks a linked list of GATKReportColumn in order by name. */ -public class GATKReportColumns extends LinkedHashMap { +public class GATKReportColumns extends LinkedHashMap implements Iterable { private List columnNames = new ArrayList(); /** @@ -52,4 +54,14 @@ public class GATKReportColumns extends LinkedHashMap { columnNames.add(key); return super.put(key, value); } + + @Override + public Iterator iterator() { + return new Iterator() { + int offset = 0; + public boolean hasNext() { return offset < columnNames.size() ; } + public GATKReportColumn next() { return getByIndex(offset++); } + public void remove() { throw new UnsupportedOperationException("Cannot remove from a GATKReportColumn iterator"); } + }; + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java index 3e3aa29a7..2fd5ad7e3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java @@ -286,6 +286,10 @@ public class GATKReportTable { } } + public boolean containsKey(Object primaryKey) { + return primaryKeyColumn.contains(primaryKey); + } + /** * Set the value for a given position in the table * diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportVersion.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportVersion.java old mode 100644 new mode 100755 diff --git a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java index 27fd173cb..c6321e2ad 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/traversals/TraversalEngine.java @@ -358,7 +358,7 @@ public abstract class TraversalEngine,Provide public void printOnTraversalDone() { printProgress(null, null, true); - final double elapsed = timer.getElapsedTime(); + final double elapsed = timer == null ? 0 : timer.getElapsedTime(); ReadMetrics cumulativeMetrics = engine.getCumulativeMetrics(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintRODsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintRODsWalker.java index 84549b13a..7960f5c35 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintRODsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintRODsWalker.java @@ -26,21 +26,23 @@ package org.broadinstitute.sting.gatk.walkers; import org.broad.tribble.Feature; +import org.broadinstitute.sting.commandline.Input; import org.broadinstitute.sting.commandline.Output; +import org.broadinstitute.sting.commandline.RodBinding; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.refdata.VariantContextAdaptors; -import org.broadinstitute.sting.gatk.refdata.utils.GATKFeature; import java.io.PrintStream; -import java.util.Iterator; /** * Prints out all of the RODs in the input data set. Data is rendered using the toString() method * of the given ROD. */ public class PrintRODsWalker extends RodWalker { + @Input(fullName="input", shortName = "input", doc="The input ROD which should be printed out.", required=true) + public RodBinding input; + @Output PrintStream out; @@ -62,7 +64,7 @@ public class PrintRODsWalker extends RodWalker { if ( tracker == null ) return 0; - for ( Feature feature : tracker.getValues(Feature.class) ) { + for ( Feature feature : tracker.getValues(Feature.class, context.getLocation()) ) { out.println(feature.toString()); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java index fdfac6bf7..4f072e88c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/PrintReadsWalker.java @@ -68,6 +68,13 @@ import org.broadinstitute.sting.gatk.refdata.ReadMetaDataTracker; * -I input1.bam \ * -I input2.bam \ * --read_filter MappingQualityZero + * + * java -Xmx2g -jar GenomeAnalysisTK.jar \ + * -R ref.fasta \ + * -T PrintReads \ + * -o output.bam \ + * -I input.bam \ + * -n 2000 * * */ diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java index c88c7c3c4..10261112c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.walkers; +import net.sf.samtools.SAMSequenceDictionary; import org.apache.log4j.Logger; import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; @@ -77,6 +78,15 @@ public abstract class Walker { return toolkit; } + /** + * Gets the master sequence dictionary for this walker + * @link GenomeAnalysisEngine.getMasterSequenceDictionary + * @return + */ + protected SAMSequenceDictionary getMasterSequenceDictionary() { + return getToolkit().getMasterSequenceDictionary(); + } + /** * (conceptual static) method that states whether you want to see reads piling up at a locus * that contain a deletion at the locus. diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java index cf68a9121..e501258c5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalance.java @@ -43,6 +43,9 @@ import java.util.List; import java.util.Map; +/** + * The allele balance (fraction of ref bases over ref + alt bases) across all bialleleic het-called samples + */ public class AlleleBalance extends InfoFieldAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java index ddb7ab828..75c4037d5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AlleleBalanceBySample.java @@ -16,6 +16,9 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.*; +/** + * The allele balance (fraction of ref bases over ref + alt bases) separately for each bialleleic het-called sample + */ public class AlleleBalanceBySample extends GenotypeAnnotation implements ExperimentalAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, AlignmentContext stratifiedContext, VariantContext vc, Genotype g) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AnnotationByDepth.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AnnotationByDepth.java index dc41dbc81..353fd1c2c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AnnotationByDepth.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/AnnotationByDepth.java @@ -6,8 +6,9 @@ import org.broadinstitute.sting.utils.variantcontext.Genotype; import java.util.Map; - - +/** + * Abstract base class for all annotations that are normalized by depth + */ public abstract class AnnotationByDepth extends InfoFieldAnnotation { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java index ecfd9b707..46aa6d0f3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseCounts.java @@ -47,6 +47,9 @@ import java.util.List; import java.util.Map; +/** + * Count of A, C, G, T bases across all samples + */ public class BaseCounts extends InfoFieldAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java index 2a5c996f7..6cab6d95f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/BaseQualityRankSumTest.java @@ -13,6 +13,9 @@ import java.util.LinkedHashMap; import java.util.List; +/** + * The phred-scaled p-value (u-based z-approximation) from the Mann-Whitney Rank Sum Test for base qualities (ref bases vs. bases of the alternate allele) + */ public class BaseQualityRankSumTest extends RankSumTest { public List getKeyNames() { return Arrays.asList("BaseQRankSum"); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java index ad06dcf52..5ed2a6761 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ChromosomeCounts.java @@ -44,6 +44,11 @@ import java.util.List; import java.util.Map; +/** + * Allele count in genotypes, for each ALT allele, in the same order as listed; + * allele Frequency, for each ALT allele, in the same order as listed; total number + * of alleles in called genotypes. + */ public class ChromosomeCounts extends InfoFieldAnnotation implements StandardAnnotation { private String[] keyNames = { VCFConstants.ALLELE_NUMBER_KEY, VCFConstants.ALLELE_COUNT_KEY, VCFConstants.ALLELE_FREQUENCY_KEY }; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java index a4d8db5bd..864be55b7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthOfCoverage.java @@ -16,7 +16,23 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - +/** + * Total (unfiltered) depth over all samples. + * + * This and AD are complementary fields that are two important ways of thinking about the depth of the data for this sample + * at this site. The DP field describe the total depth of reads that passed the Unified Genotypers internal + * quality control metrics (like MAPQ > 17, for example), whatever base was present in the read at this site. + * The AD values (one for each of REF and ALT fields) is the count of all reads that carried with them the + * REF and ALT alleles. The reason for this distinction is that the DP is in some sense reflective of the + * power I have to determine the genotype of the sample at this site, while the AD tells me how many times + * I saw each of the REF and ALT alleles in the reads, free of any bias potentially introduced by filtering + * the reads. If, for example, I believe there really is a an A/T polymorphism at a site, then I would like + * to know the counts of A and T bases in this sample, even for reads with poor mapping quality that would + * normally be excluded from the statistical calculations going into GQ and QUAL. + * + * Note that the DP is affected by downsampling (-dcov) though, so the max value one can obtain for N samples with + * -dcov D is N * D + */ public class DepthOfCoverage extends InfoFieldAnnotation implements StandardAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java index 1652c8de7..5d706d9c5 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/DepthPerAlleleBySample.java @@ -23,6 +23,25 @@ import java.util.List; import java.util.Map; +/** + * The depth of coverage of each VCF allele in this sample. + * + * This and DP are complementary fields that are two important ways of thinking about the depth of the data for this sample + * at this site. The DP field describe the total depth of reads that passed the Unified Genotypers internal + * quality control metrics (like MAPQ > 17, for example), whatever base was present in the read at this site. + * The AD values (one for each of REF and ALT fields) is the count of all reads that carried with them the + * REF and ALT alleles. The reason for this distinction is that the DP is in some sense reflective of the + * power I have to determine the genotype of the sample at this site, while the AD tells me how many times + * I saw each of the REF and ALT alleles in the reads, free of any bias potentially introduced by filtering + * the reads. If, for example, I believe there really is a an A/T polymorphism at a site, then I would like + * to know the counts of A and T bases in this sample, even for reads with poor mapping quality that would + * normally be excluded from the statistical calculations going into GQ and QUAL. Please note, however, that + * the AD isn't necessarily calculated exactly for indels (it counts as non-reference only those indels that + * are actually present and correctly left-aligned in the alignments themselves). Because of this fact and + * because the AD includes reads and bases that were filtered by the Unified Genotyper, one should not base + * assumptions about the underlying genotype based on it; instead, the genotype likelihoods (PLs) are what + * determine the genotype calls (see below). + */ public class DepthPerAlleleBySample extends GenotypeAnnotation implements StandardAnnotation { private static String REF_ALLELE = "REF"; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java index 0cfca48fa..393eb549c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/FisherStrand.java @@ -43,6 +43,11 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.*; +/** + * Phred-scaled p-value using Fisher's Exact Test to detect strand bias (the variation + * being seen on only the forward or only the reverse strand) in the reads? More bias is + * indicative of false positive calls. + */ public class FisherStrand extends InfoFieldAnnotation implements StandardAnnotation { private static final String FS = "FS"; private static final double MIN_PVALUE = 1E-320; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java index a46473f60..11a64b49f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GCContent.java @@ -17,6 +17,9 @@ import java.util.List; import java.util.Map; +/** + * The GC content (# GC bases / # all bases) of the reference within 50 bp +/- this site + */ public class GCContent extends InfoFieldAnnotation implements ExperimentalAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java index 9af3b8e8e..c142109fa 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HaplotypeScore.java @@ -34,12 +34,12 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnot import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.StandardAnnotation; import org.broadinstitute.sting.gatk.walkers.genotyper.IndelGenotypeLikelihoodsCalculationModel; import org.broadinstitute.sting.utils.BaseUtils; +import org.broadinstitute.sting.utils.Haplotype; import org.broadinstitute.sting.utils.MathUtils; import org.broadinstitute.sting.utils.QualityUtils; import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.genotype.Haplotype; import org.broadinstitute.sting.utils.pileup.PileupElement; import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; import org.broadinstitute.sting.utils.sam.AlignmentUtils; @@ -49,6 +49,10 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.*; +/** + * Consistency of the site with two (and only two) segregating haplotypes. Higher scores + * are indicative of regions with bad alignments, often leading to artifactual SNP and indel calls. + */ public class HaplotypeScore extends InfoFieldAnnotation implements StandardAnnotation { private final static boolean DEBUG = false; private final static int MIN_CONTEXT_WING_SIZE = 10; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java index 045505698..f068ed895 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HardyWeinberg.java @@ -19,6 +19,9 @@ import java.util.List; import java.util.Map; +/** + * Phred-scaled P value of genotype-based (using GT field) test for Hardy-Weinberg test for disequilibrium + */ public class HardyWeinberg extends InfoFieldAnnotation implements WorkInProgressAnnotation { private static final int MIN_SAMPLES = 10; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java index 463f7a645..197a00243 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/HomopolymerRun.java @@ -16,7 +16,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - +/** + * Largest contiguous homopolymer run of the variant allele in either direction on the reference. + */ public class HomopolymerRun extends InfoFieldAnnotation implements StandardAnnotation { private boolean ANNOTATE_INDELS = true; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GLstats.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java similarity index 90% rename from public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GLstats.java rename to public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java index 5295d6d21..a14007147 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/GLstats.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/InbreedingCoeff.java @@ -17,14 +17,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * Created by IntelliJ IDEA. - * User: rpoplin - * Date: 5/16/11 - */ -// A set of annotations calculated directly from the GLs -public class GLstats extends InfoFieldAnnotation implements StandardAnnotation { +/** + * Likelihood-based (using PL field) test for the inbreeding among samples. + * + * A continuous generalization of the Hardy-Weinberg test for disequilibrium that works + * well with limited coverage per sample. See the 1000 Genomes Phase I release for + * more information. + */ +public class InbreedingCoeff extends InfoFieldAnnotation implements StandardAnnotation { private static final int MIN_SAMPLES = 10; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java index bfede40d2..e0abfcf3c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/IndelType.java @@ -14,11 +14,7 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.*; /** - * Created by IntelliJ IDEA. - * User: delangel - * Date: Mar 11, 2011 - * Time: 11:47:33 AM - * To change this template use File | Settings | File Templates. + * Rough category of indel type (insertion, deletion, multi-allelic, other) */ public class IndelType extends InfoFieldAnnotation implements ExperimentalAnnotation { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java index 09ffe0fb6..753740258 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/LowMQ.java @@ -17,6 +17,9 @@ import java.util.List; import java.util.Map; +/** + * Triplet annotation: fraction of MAQP == 0, MAPQ < 10, and count of all mapped reads + */ public class LowMQ extends InfoFieldAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java index cc62580a9..157c615d7 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityRankSumTest.java @@ -14,6 +14,9 @@ import java.util.LinkedHashMap; import java.util.List; +/** + * The phred-scaled p-value (u-based z-approximation) from the Mann-Whitney Rank Sum Test for mapping qualities (reads with ref bases vs. those with the alternate allele) + */ public class MappingQualityRankSumTest extends RankSumTest { public List getKeyNames() { return Arrays.asList("MQRankSum"); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java index f9caae227..3a3efc4e8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZero.java @@ -19,6 +19,9 @@ import java.util.List; import java.util.Map; +/** + * Total count across all samples of mapping quality zero reads + */ public class MappingQualityZero extends InfoFieldAnnotation implements StandardAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java index 3d234a1e3..f14d7a8a5 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroBySample.java @@ -1,85 +1,81 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.gatk.walkers.annotator; - -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; -import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; -import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine; -import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; -import org.broadinstitute.sting.utils.pileup.PileupElement; -import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; -import org.broadinstitute.sting.utils.variantcontext.Genotype; -import org.broadinstitute.sting.utils.variantcontext.VariantContext; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by IntelliJ IDEA. - * User: asivache - * Date: Feb 4, 2011 - * Time: 6:46:25 PM - * To change this template use File | Settings | File Templates. - */ -public class MappingQualityZeroBySample extends GenotypeAnnotation { - public Map annotate(RefMetaDataTracker tracker, - AnnotatorCompatibleWalker walker, ReferenceContext ref, AlignmentContext context, VariantContext vc, Genotype g) { - if ( g == null || !g.isCalled() ) - return null; - - int mq0 = 0; - ReadBackedPileup pileup = null; - if (vc.isIndel() && context.hasExtendedEventPileup()) - pileup = context.getExtendedEventPileup(); - else if (context.hasBasePileup()) - pileup = context.getBasePileup(); - else return null; - - if (pileup != null) { - for (PileupElement p : pileup ) { - if ( p.getMappingQual() == 0 ) - mq0++; - } - } - Map map = new HashMap(); - map.put(getKeyNames().get(0), String.format("%d", mq0)); - return map; - } - - public List getKeyNames() { return Arrays.asList(VCFConstants.MAPPING_QUALITY_ZERO_KEY); } - - public List getDescriptions() { return Arrays.asList( - new VCFFormatHeaderLine(getKeyNames().get(0), 1, - VCFHeaderLineType.Integer, "Number of Mapping Quality Zero Reads per sample")); } - - -} +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.annotator; + +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; +import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants; +import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; +import org.broadinstitute.sting.utils.variantcontext.Genotype; +import org.broadinstitute.sting.utils.variantcontext.VariantContext; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Count for each sample of mapping quality zero reads + */ +public class MappingQualityZeroBySample extends GenotypeAnnotation { + public Map annotate(RefMetaDataTracker tracker, + AnnotatorCompatibleWalker walker, ReferenceContext ref, AlignmentContext context, VariantContext vc, Genotype g) { + if ( g == null || !g.isCalled() ) + return null; + + int mq0 = 0; + ReadBackedPileup pileup = null; + if (vc.isIndel() && context.hasExtendedEventPileup()) + pileup = context.getExtendedEventPileup(); + else if (context.hasBasePileup()) + pileup = context.getBasePileup(); + else return null; + + if (pileup != null) { + for (PileupElement p : pileup ) { + if ( p.getMappingQual() == 0 ) + mq0++; + } + } + Map map = new HashMap(); + map.put(getKeyNames().get(0), String.format("%d", mq0)); + return map; + } + + public List getKeyNames() { return Arrays.asList(VCFConstants.MAPPING_QUALITY_ZERO_KEY); } + + public List getDescriptions() { return Arrays.asList( + new VCFFormatHeaderLine(getKeyNames().get(0), 1, + VCFHeaderLineType.Integer, "Number of Mapping Quality Zero Reads per sample")); } + + +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java index 3e8fe8998..2164537b8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/MappingQualityZeroFraction.java @@ -17,8 +17,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - - +/** + * Fraction of all reads across samples that have mapping quality zero + */ public class MappingQualityZeroFraction extends InfoFieldAnnotation implements ExperimentalAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java index 74c562045..891e9ae56 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/NBaseCount.java @@ -17,11 +17,8 @@ import java.util.List; import java.util.Map; /** - * Created by IntelliJ IDEA. - * User: rpoplin - * Date: 5/16/11 + * The number of N bases, counting only SOLiD data */ - public class NBaseCount extends InfoFieldAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { if( stratifiedContexts.size() == 0 ) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java index 9a292c39a..552289309 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/QualByDepth.java @@ -1,5 +1,6 @@ package org.broadinstitute.sting.gatk.walkers.annotator; +import org.broadinstitute.sting.commandline.Hidden; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; @@ -15,7 +16,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - +/** + * Variant confidence (given as (AB+BB)/AA from the PLs) / unfiltered depth. + * + * Low scores are indicative of false positive calls and artifacts. + */ public class QualByDepth extends AnnotationByDepth implements StandardAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java index 668129888..40f6d20d3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RMSMappingQuality.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.Map; +/** + * Root Mean Square of the mapping quality of the reads across all samples. + */ public class RMSMappingQuality extends InfoFieldAnnotation implements StandardAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java index 52c704055..93e093248 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/RankSumTest.java @@ -21,7 +21,9 @@ import java.util.List; import java.util.Map; - +/** + * Abstract root for all RankSum based annotations + */ public abstract class RankSumTest extends InfoFieldAnnotation implements StandardAnnotation { static final double INDEL_LIKELIHOOD_THRESH = 0.1; static final boolean DEBUG = false; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadDepthAndAllelicFractionBySample.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadDepthAndAllelicFractionBySample.java index 26ca08380..772541eb6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadDepthAndAllelicFractionBySample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadDepthAndAllelicFractionBySample.java @@ -1,209 +1,207 @@ -/* - * Copyright (c) 2010 The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.gatk.walkers.annotator; - -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; -import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine; -import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount; -import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; -import org.broadinstitute.sting.utils.pileup.ExtendedEventPileupElement; -import org.broadinstitute.sting.utils.pileup.PileupElement; -import org.broadinstitute.sting.utils.pileup.ReadBackedExtendedEventPileup; -import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; -import org.broadinstitute.sting.utils.variantcontext.Allele; -import org.broadinstitute.sting.utils.variantcontext.Genotype; -import org.broadinstitute.sting.utils.variantcontext.VariantContext; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by IntelliJ IDEA. - * User: asivache - * Date: Feb 4, 2011 - * Time: 3:59:27 PM - * To change this template use File | Settings | File Templates. - */ -public class ReadDepthAndAllelicFractionBySample extends GenotypeAnnotation { - - private static String REF_ALLELE = "REF"; - - private static String DEL = "DEL"; // constant, for speed: no need to create a key string for deletion allele every time - - public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, - AlignmentContext stratifiedContext, VariantContext vc, Genotype g) { - if ( g == null || !g.isCalled() ) - return null; - - if ( vc.isSNP() ) - return annotateSNP(stratifiedContext, vc); - if ( vc.isIndel() ) - return annotateIndel(stratifiedContext, vc); - - return null; - } - - private Map annotateSNP(AlignmentContext stratifiedContext, VariantContext vc) { - - if ( ! stratifiedContext.hasBasePileup() ) return null; - - HashMap alleleCounts = new HashMap(); - for ( Allele allele : vc.getAlternateAlleles() ) - alleleCounts.put(allele.getBases()[0], 0); - - ReadBackedPileup pileup = stratifiedContext.getBasePileup(); - int totalDepth = pileup.size(); - - Map map = new HashMap(); - map.put(getKeyNames().get(0), totalDepth); // put total depth in right away - - if ( totalDepth == 0 ) return map; // done, can not compute FA at 0 coverage!! - - int mq0 = 0; // number of "ref" reads that are acually mq0 - for ( PileupElement p : pileup ) { - if ( p.getMappingQual() == 0 ) { - mq0++; - continue; - } - if ( alleleCounts.containsKey(p.getBase()) ) // non-mq0 read and it's an alt - alleleCounts.put(p.getBase(), alleleCounts.get(p.getBase())+1); - } - - if ( mq0 == totalDepth ) return map; // if all reads are mq0, there is nothing left to do - - // we need to add counts in the correct order - String[] fracs = new String[alleleCounts.size()]; - for (int i = 0; i < vc.getAlternateAlleles().size(); i++) { - fracs[i] = String.format("%.3f", ((float)alleleCounts.get(vc.getAlternateAllele(i).getBases()[0]))/(totalDepth-mq0)); - } - - map.put(getKeyNames().get(1), fracs); - return map; - } - - private Map annotateIndel(AlignmentContext - stratifiedContext, VariantContext - vc) { - - if ( ! stratifiedContext.hasExtendedEventPileup() ) { - return null; - } - - ReadBackedExtendedEventPileup pileup = stratifiedContext.getExtendedEventPileup(); - if ( pileup == null ) - return null; - int totalDepth = pileup.size(); - - Map map = new HashMap(); - map.put(getKeyNames().get(0), totalDepth); // put total depth in right away - - if ( totalDepth == 0 ) return map; - int mq0 = 0; // number of "ref" reads that are acually mq0 - - HashMap alleleCounts = new HashMap(); - Allele refAllele = vc.getReference(); - - for ( Allele allele : vc.getAlternateAlleles() ) { - - if ( allele.isNoCall() ) { - continue; // this does not look so good, should we die??? - } - - alleleCounts.put(getAlleleRepresentation(allele), 0); - } - - for ( ExtendedEventPileupElement e : pileup.toExtendedIterable() ) { - - if ( e.getMappingQual() == 0 ) { - mq0++; - continue; - } - - if ( e.isInsertion() ) { - - final String b = e.getEventBases(); - if ( alleleCounts.containsKey(b) ) { - alleleCounts.put(b, alleleCounts.get(b)+1); - } - - } else { - if ( e.isDeletion() ) { - if ( e.getEventLength() == refAllele.length() ) { - // this is indeed the deletion allele recorded in VC - final String b = DEL; - if ( alleleCounts.containsKey(b) ) { - alleleCounts.put(b, alleleCounts.get(b)+1); - } - } -// else { -// System.out.print(" deletion of WRONG length found"); -// } - } - } - } - - if ( mq0 == totalDepth ) return map; - - String[] fracs = new String[alleleCounts.size()]; - for (int i = 0; i < vc.getAlternateAlleles().size(); i++) - fracs[i] = String.format("%.3f", - ((float)alleleCounts.get(getAlleleRepresentation(vc.getAlternateAllele(i))))/(totalDepth-mq0)); - - map.put(getKeyNames().get(1), fracs); - - //map.put(getKeyNames().get(0), counts); - return map; - } - - private String getAlleleRepresentation(Allele allele) { - if ( allele.isNull() ) { // deletion wrt the ref - return DEL; - } else { // insertion, pass actual bases - return allele.getBaseString(); - } - - } - - // public String getIndelBases() - public List getKeyNames() { return Arrays.asList("DP","FA"); } - - public List getDescriptions() { - return Arrays.asList(new VCFFormatHeaderLine(getKeyNames().get(0), - 1, - VCFHeaderLineType.Integer, - "Total read depth per sample, including MQ0"), - new VCFFormatHeaderLine(getKeyNames().get(1), - VCFHeaderLineCount.UNBOUNDED, - VCFHeaderLineType.Float, - "Fractions of reads (excluding MQ0 from both ref and alt) supporting each reported alternative allele, per sample")); - } -} +/* + * Copyright (c) 2010 The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.gatk.walkers.annotator; + +import org.broadinstitute.sting.commandline.Hidden; +import org.broadinstitute.sting.gatk.contexts.AlignmentContext; +import org.broadinstitute.sting.gatk.contexts.ReferenceContext; +import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; +import org.broadinstitute.sting.utils.codecs.vcf.VCFFormatHeaderLine; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineCount; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; +import org.broadinstitute.sting.utils.pileup.ExtendedEventPileupElement; +import org.broadinstitute.sting.utils.pileup.PileupElement; +import org.broadinstitute.sting.utils.pileup.ReadBackedExtendedEventPileup; +import org.broadinstitute.sting.utils.pileup.ReadBackedPileup; +import org.broadinstitute.sting.utils.variantcontext.Allele; +import org.broadinstitute.sting.utils.variantcontext.Genotype; +import org.broadinstitute.sting.utils.variantcontext.VariantContext; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Unsupported + */ +@Hidden +public class ReadDepthAndAllelicFractionBySample extends GenotypeAnnotation { + + private static String REF_ALLELE = "REF"; + + private static String DEL = "DEL"; // constant, for speed: no need to create a key string for deletion allele every time + + public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, + AlignmentContext stratifiedContext, VariantContext vc, Genotype g) { + if ( g == null || !g.isCalled() ) + return null; + + if ( vc.isSNP() ) + return annotateSNP(stratifiedContext, vc); + if ( vc.isIndel() ) + return annotateIndel(stratifiedContext, vc); + + return null; + } + + private Map annotateSNP(AlignmentContext stratifiedContext, VariantContext vc) { + + if ( ! stratifiedContext.hasBasePileup() ) return null; + + HashMap alleleCounts = new HashMap(); + for ( Allele allele : vc.getAlternateAlleles() ) + alleleCounts.put(allele.getBases()[0], 0); + + ReadBackedPileup pileup = stratifiedContext.getBasePileup(); + int totalDepth = pileup.size(); + + Map map = new HashMap(); + map.put(getKeyNames().get(0), totalDepth); // put total depth in right away + + if ( totalDepth == 0 ) return map; // done, can not compute FA at 0 coverage!! + + int mq0 = 0; // number of "ref" reads that are acually mq0 + for ( PileupElement p : pileup ) { + if ( p.getMappingQual() == 0 ) { + mq0++; + continue; + } + if ( alleleCounts.containsKey(p.getBase()) ) // non-mq0 read and it's an alt + alleleCounts.put(p.getBase(), alleleCounts.get(p.getBase())+1); + } + + if ( mq0 == totalDepth ) return map; // if all reads are mq0, there is nothing left to do + + // we need to add counts in the correct order + String[] fracs = new String[alleleCounts.size()]; + for (int i = 0; i < vc.getAlternateAlleles().size(); i++) { + fracs[i] = String.format("%.3f", ((float)alleleCounts.get(vc.getAlternateAllele(i).getBases()[0]))/(totalDepth-mq0)); + } + + map.put(getKeyNames().get(1), fracs); + return map; + } + + private Map annotateIndel(AlignmentContext + stratifiedContext, VariantContext + vc) { + + if ( ! stratifiedContext.hasExtendedEventPileup() ) { + return null; + } + + ReadBackedExtendedEventPileup pileup = stratifiedContext.getExtendedEventPileup(); + if ( pileup == null ) + return null; + int totalDepth = pileup.size(); + + Map map = new HashMap(); + map.put(getKeyNames().get(0), totalDepth); // put total depth in right away + + if ( totalDepth == 0 ) return map; + int mq0 = 0; // number of "ref" reads that are acually mq0 + + HashMap alleleCounts = new HashMap(); + Allele refAllele = vc.getReference(); + + for ( Allele allele : vc.getAlternateAlleles() ) { + + if ( allele.isNoCall() ) { + continue; // this does not look so good, should we die??? + } + + alleleCounts.put(getAlleleRepresentation(allele), 0); + } + + for ( ExtendedEventPileupElement e : pileup.toExtendedIterable() ) { + + if ( e.getMappingQual() == 0 ) { + mq0++; + continue; + } + + if ( e.isInsertion() ) { + + final String b = e.getEventBases(); + if ( alleleCounts.containsKey(b) ) { + alleleCounts.put(b, alleleCounts.get(b)+1); + } + + } else { + if ( e.isDeletion() ) { + if ( e.getEventLength() == refAllele.length() ) { + // this is indeed the deletion allele recorded in VC + final String b = DEL; + if ( alleleCounts.containsKey(b) ) { + alleleCounts.put(b, alleleCounts.get(b)+1); + } + } +// else { +// System.out.print(" deletion of WRONG length found"); +// } + } + } + } + + if ( mq0 == totalDepth ) return map; + + String[] fracs = new String[alleleCounts.size()]; + for (int i = 0; i < vc.getAlternateAlleles().size(); i++) + fracs[i] = String.format("%.3f", + ((float)alleleCounts.get(getAlleleRepresentation(vc.getAlternateAllele(i))))/(totalDepth-mq0)); + + map.put(getKeyNames().get(1), fracs); + + //map.put(getKeyNames().get(0), counts); + return map; + } + + private String getAlleleRepresentation(Allele allele) { + if ( allele.isNull() ) { // deletion wrt the ref + return DEL; + } else { // insertion, pass actual bases + return allele.getBaseString(); + } + + } + + // public String getIndelBases() + public List getKeyNames() { return Arrays.asList("DP","FA"); } + + public List getDescriptions() { + return Arrays.asList(new VCFFormatHeaderLine(getKeyNames().get(0), + 1, + VCFHeaderLineType.Integer, + "Total read depth per sample, including MQ0"), + new VCFFormatHeaderLine(getKeyNames().get(1), + VCFHeaderLineCount.UNBOUNDED, + VCFHeaderLineType.Float, + "Fractions of reads (excluding MQ0 from both ref and alt) supporting each reported alternative allele, per sample")); + } +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java index aabfb2970..27a9306d4 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/ReadPosRankSumTest.java @@ -19,11 +19,8 @@ import java.util.LinkedHashMap; import java.util.List; /** - * Created by IntelliJ IDEA. - * User: rpoplin - * Date: 3/30/11 + * The phred-scaled p-value (u-based z-approximation) from the Mann-Whitney Rank Sum Test for the distance from the end of the read for reads with the alternate allele; if the alternate allele is only seen near the ends of reads this is indicative of error). */ - public class ReadPosRankSumTest extends RankSumTest { public List getKeyNames() { return Arrays.asList("ReadPosRankSum"); } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SBByDepth.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SBByDepth.java index 180bed24d..131b87794 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SBByDepth.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SBByDepth.java @@ -15,8 +15,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - - +/** + * SB annotation value by depth of alt containing samples + */ public class SBByDepth extends AnnotationByDepth { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { @@ -26,7 +27,7 @@ public class SBByDepth extends AnnotationByDepth { if (!vc.hasAttribute(VCFConstants.STRAND_BIAS_KEY)) return null; - double sBias = Double.valueOf(vc.getAttributeAsString(VCFConstants.STRAND_BIAS_KEY)); + double sBias = vc.getAttributeAsDouble(VCFConstants.STRAND_BIAS_KEY, -1); final Map genotypes = vc.getGenotypes(); if ( genotypes == null || genotypes.size() == 0 ) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java index cd396036f..ff409484d 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SampleList.java @@ -41,7 +41,9 @@ import java.util.HashMap; import java.util.List; import java.util.Map; - +/** + * List all of the samples in the info field + */ public class SampleList extends InfoFieldAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java index 350c683c2..973b3277d 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SnpEff.java @@ -24,7 +24,9 @@ package org.broadinstitute.sting.gatk.walkers.annotator; +import org.apache.log4j.Logger; import org.broadinstitute.sting.commandline.RodBinding; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; @@ -32,10 +34,7 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompa import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.ExperimentalAnnotation; import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants; -import org.broadinstitute.sting.utils.codecs.snpEff.SnpEffFeature; -import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLineType; -import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; +import org.broadinstitute.sting.utils.codecs.vcf.*; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.variantcontext.VariantContext; @@ -46,134 +45,522 @@ import java.util.*; * (http://snpeff.sourceforge.net/). * * For each variant, chooses one of the effects of highest biological impact from the SnpEff - * output file (which must be provided on the command line via --snpEffFile:SnpEff ), + * output file (which must be provided on the command line via --snpEffFile filename.vcf), * and adds annotations on that effect. * - * The possible biological effects and their associated impacts are defined in the class: - * org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants - * * @author David Roazen */ public class SnpEff extends InfoFieldAnnotation implements ExperimentalAnnotation { - // SnpEff annotation key names: - public static final String GENE_ID_KEY = "GENE_ID"; - public static final String GENE_NAME_KEY = "GENE_NAME"; - public static final String TRANSCRIPT_ID_KEY = "TRANSCRIPT_ID"; - public static final String EXON_ID_KEY = "EXON_ID"; - public static final String EXON_RANK_KEY = "EXON_RANK"; - public static final String WITHIN_NON_CODING_GENE_KEY = "WITHIN_NON_CODING_GENE"; - public static final String EFFECT_KEY = "EFFECT"; - public static final String EFFECT_IMPACT_KEY = "EFFECT_IMPACT"; - public static final String EFFECT_EXTRA_INFORMATION_KEY = "EFFECT_EXTRA_INFORMATION"; - public static final String OLD_NEW_AA_KEY = "OLD_NEW_AA"; - public static final String OLD_NEW_CODON_KEY = "OLD_NEW_CODON"; - public static final String CODON_NUM_KEY = "CODON_NUM"; - public static final String CDS_SIZE_KEY = "CDS_SIZE"; + private static Logger logger = Logger.getLogger(SnpEff.class); + + // We refuse to parse SnpEff output files generated by unsupported versions, or + // lacking a SnpEff version number in the VCF header: + public static final String[] SUPPORTED_SNPEFF_VERSIONS = { "2.0.2" }; + public static final String SNPEFF_VCF_HEADER_VERSION_LINE_KEY = "SnpEffVersion"; + public static final String SNPEFF_VCF_HEADER_COMMAND_LINE_KEY = "SnpEffCmd"; + + // When we write the SnpEff version number and command line to the output VCF, we change + // the key name slightly so that the output VCF won't be confused in the future for an + // output file produced by SnpEff directly: + public static final String OUTPUT_VCF_HEADER_VERSION_LINE_KEY = "Original" + SNPEFF_VCF_HEADER_VERSION_LINE_KEY; + public static final String OUTPUT_VCF_HEADER_COMMAND_LINE_KEY = "Original" + SNPEFF_VCF_HEADER_COMMAND_LINE_KEY; + + // SnpEff aggregates all effects (and effect metadata) together into a single INFO + // field annotation with the key EFF: + public static final String SNPEFF_INFO_FIELD_KEY = "EFF"; + public static final String SNPEFF_EFFECT_METADATA_DELIMITER = "[()]"; + public static final String SNPEFF_EFFECT_METADATA_SUBFIELD_DELIMITER = "\\|"; + + // Key names for the INFO field annotations we will add to each record, along + // with parsing-related information: + public enum InfoFieldKey { + EFFECT_KEY ("SNPEFF_EFFECT", -1), + IMPACT_KEY ("SNPEFF_IMPACT", 0), + CODON_CHANGE_KEY ("SNPEFF_CODON_CHANGE", 1), + AMINO_ACID_CHANGE_KEY ("SNPEFF_AMINO_ACID_CHANGE", 2), + GENE_NAME_KEY ("SNPEFF_GENE_NAME", 3), + GENE_BIOTYPE_KEY ("SNPEFF_GENE_BIOTYPE", 4), + TRANSCRIPT_ID_KEY ("SNPEFF_TRANSCRIPT_ID", 6), + EXON_ID_KEY ("SNPEFF_EXON_ID", 7), + FUNCTIONAL_CLASS_KEY ("SNPEFF_FUNCTIONAL_CLASS", -1); + + // Actual text of the key + private final String keyName; + + // Index within the effect metadata subfields from the SnpEff EFF annotation + // where each key's associated value can be found during parsing. + private final int fieldIndex; + + InfoFieldKey ( String keyName, int fieldIndex ) { + this.keyName = keyName; + this.fieldIndex = fieldIndex; + } + + public String getKeyName() { + return keyName; + } + + public int getFieldIndex() { + return fieldIndex; + } + } + + // Possible SnpEff biological effects. All effect names found in the SnpEff input file + // are validated against this list. + public enum EffectType { + // High-impact effects: + FRAME_SHIFT (EffectFunctionalClass.NONE, false), + STOP_GAINED (EffectFunctionalClass.NONSENSE, false), + START_LOST (EffectFunctionalClass.NONE, false), + SPLICE_SITE_ACCEPTOR (EffectFunctionalClass.NONE, false), + SPLICE_SITE_DONOR (EffectFunctionalClass.NONE, false), + EXON_DELETED (EffectFunctionalClass.NONE, false), + STOP_LOST (EffectFunctionalClass.NONE, false), + + // Moderate-impact effects: + NON_SYNONYMOUS_CODING (EffectFunctionalClass.MISSENSE, false), + CODON_CHANGE (EffectFunctionalClass.NONE, false), + CODON_INSERTION (EffectFunctionalClass.NONE, false), + CODON_CHANGE_PLUS_CODON_INSERTION (EffectFunctionalClass.NONE, false), + CODON_DELETION (EffectFunctionalClass.NONE, false), + CODON_CHANGE_PLUS_CODON_DELETION (EffectFunctionalClass.NONE, false), + UTR_5_DELETED (EffectFunctionalClass.NONE, false), + UTR_3_DELETED (EffectFunctionalClass.NONE, false), + + // Low-impact effects: + SYNONYMOUS_CODING (EffectFunctionalClass.SILENT, false), + SYNONYMOUS_START (EffectFunctionalClass.SILENT, false), + NON_SYNONYMOUS_START (EffectFunctionalClass.SILENT, false), + SYNONYMOUS_STOP (EffectFunctionalClass.SILENT, false), + NON_SYNONYMOUS_STOP (EffectFunctionalClass.SILENT, false), + START_GAINED (EffectFunctionalClass.NONE, false), + + // Modifiers: + NONE (EffectFunctionalClass.NONE, true), + CHROMOSOME (EffectFunctionalClass.NONE, true), + INTERGENIC (EffectFunctionalClass.NONE, true), + UPSTREAM (EffectFunctionalClass.NONE, true), + UTR_5_PRIME (EffectFunctionalClass.NONE, true), + CDS (EffectFunctionalClass.NONE, true), + GENE (EffectFunctionalClass.NONE, true), + TRANSCRIPT (EffectFunctionalClass.NONE, true), + EXON (EffectFunctionalClass.NONE, true), + INTRON (EffectFunctionalClass.NONE, true), + UTR_3_PRIME (EffectFunctionalClass.NONE, true), + DOWNSTREAM (EffectFunctionalClass.NONE, true), + INTRON_CONSERVED (EffectFunctionalClass.NONE, true), + INTERGENIC_CONSERVED (EffectFunctionalClass.NONE, true), + REGULATION (EffectFunctionalClass.NONE, true), + CUSTOM (EffectFunctionalClass.NONE, true), + WITHIN_NON_CODING_GENE (EffectFunctionalClass.NONE, true); + + private final EffectFunctionalClass functionalClass; + private final boolean isModifier; + + EffectType ( EffectFunctionalClass functionalClass, boolean isModifier ) { + this.functionalClass = functionalClass; + this.isModifier = isModifier; + } + + public EffectFunctionalClass getFunctionalClass() { + return functionalClass; + } + + public boolean isModifier() { + return isModifier; + } + } + + // SnpEff labels each effect as either LOW, MODERATE, or HIGH impact. We take the additional step of + // classifying some of the LOW impact effects as MODIFIERs. + public enum EffectImpact { + MODIFIER (0), + LOW (1), + MODERATE (2), + HIGH (3); + + private final int severityRating; + + EffectImpact ( int severityRating ) { + this.severityRating = severityRating; + } + + public boolean isHigherImpactThan ( EffectImpact other ) { + return this.severityRating > other.severityRating; + } + + public boolean isSameImpactAs ( EffectImpact other ) { + return this.severityRating == other.severityRating; + } + } + + // SnpEff labels most effects as either CODING or NON_CODING, but sometimes omits this information. + public enum EffectCoding { + CODING, + NON_CODING, + UNKNOWN + } + + // We assign a functional class to each SnpEff effect. + public enum EffectFunctionalClass { + NONE (0), + SILENT (1), + MISSENSE (2), + NONSENSE (3); + + private final int priority; + + EffectFunctionalClass ( int priority ) { + this.priority = priority; + } + + public boolean isHigherPriorityThan ( EffectFunctionalClass other ) { + return this.priority > other.priority; + } + } + + public void initialize ( AnnotatorCompatibleWalker walker, GenomeAnalysisEngine toolkit, Set headerLines ) { + // Make sure that we actually have a valid SnpEff rod binding (just in case the user specified -A SnpEff + // without providing a SnpEff rod via --snpEffFile): + validateRodBinding(walker.getSnpEffRodBinding()); + RodBinding snpEffRodBinding = walker.getSnpEffRodBinding(); + + // Make sure that the SnpEff version number and command-line header lines are present in the VCF header of + // the SnpEff rod, and that the file was generated by a supported version of SnpEff: + VCFHeader snpEffVCFHeader = VCFUtils.getVCFHeadersFromRods(toolkit, Arrays.asList(snpEffRodBinding.getName())).get(snpEffRodBinding.getName()); + VCFHeaderLine snpEffVersionLine = snpEffVCFHeader.getOtherHeaderLine(SNPEFF_VCF_HEADER_VERSION_LINE_KEY); + VCFHeaderLine snpEffCommandLine = snpEffVCFHeader.getOtherHeaderLine(SNPEFF_VCF_HEADER_COMMAND_LINE_KEY); + + checkSnpEffVersion(snpEffVersionLine); + checkSnpEffCommandLine(snpEffCommandLine); + + // If everything looks ok, add the SnpEff version number and command-line header lines to the + // header of the VCF output file, changing the key names so that our output file won't be + // mistaken in the future for a SnpEff output file: + headerLines.add(new VCFHeaderLine(OUTPUT_VCF_HEADER_VERSION_LINE_KEY, snpEffVersionLine.getValue())); + headerLines.add(new VCFHeaderLine(OUTPUT_VCF_HEADER_COMMAND_LINE_KEY, snpEffCommandLine.getValue())); + } public Map annotate ( RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc ) { - RodBinding snpEffRodBinding = walker.getSnpEffRodBinding(); - validateRodBinding(snpEffRodBinding); + RodBinding snpEffRodBinding = walker.getSnpEffRodBinding(); - List features = tracker.getValues(snpEffRodBinding, ref.getLocus()); + // Get only SnpEff records that start at this locus, not merely span it: + List snpEffRecords = tracker.getValues(snpEffRodBinding, ref.getLocus()); - // Add only annotations for one of the most biologically-significant effects as defined in - // the SnpEffConstants class: - SnpEffFeature mostSignificantEffect = getMostSignificantEffect(features); - - if ( mostSignificantEffect == null ) { + // Within this set, look for a SnpEff record whose ref/alt alleles match the record to annotate. + // If there is more than one such record, we only need to pick the first one, since the biological + // effects will be the same across all such records: + VariantContext matchingRecord = getMatchingSnpEffRecord(snpEffRecords, vc); + if ( matchingRecord == null ) { return null; } - return generateAnnotations(mostSignificantEffect); + // Parse the SnpEff INFO field annotation from the matching record into individual effect objects: + List effects = parseSnpEffRecord(matchingRecord); + if ( effects.size() == 0 ) { + return null; + } + + // Add only annotations for one of the most biologically-significant effects from this set: + SnpEffEffect mostSignificantEffect = getMostSignificantEffect(effects); + return mostSignificantEffect.getAnnotations(); } - private void validateRodBinding ( RodBinding snpEffRodBinding ) { + private void validateRodBinding ( RodBinding snpEffRodBinding ) { if ( snpEffRodBinding == null || ! snpEffRodBinding.isBound() ) { - throw new UserException("The SnpEff annotator requires that a SnpEff output file be provided " + - "as a rodbinding on the command line, but no SnpEff rodbinding was found."); + throw new UserException("The SnpEff annotator requires that a SnpEff VCF output file be provided " + + "as a rodbinding on the command line via the --snpEffFile option, but " + + "no SnpEff rodbinding was found."); } } - private SnpEffFeature getMostSignificantEffect ( List snpEffFeatures ) { - SnpEffFeature mostSignificantEffect = null; + private void checkSnpEffVersion ( VCFHeaderLine snpEffVersionLine ) { + if ( snpEffVersionLine == null || snpEffVersionLine.getValue() == null || snpEffVersionLine.getValue().trim().length() == 0 ) { + throw new UserException("Could not find a " + SNPEFF_VCF_HEADER_VERSION_LINE_KEY + " entry in the VCF header for the SnpEff " + + "input file, and so could not verify that the file was generated by a supported version of SnpEff (" + + Arrays.toString(SUPPORTED_SNPEFF_VERSIONS) + ")"); + } - for ( SnpEffFeature snpEffFeature : snpEffFeatures ) { + String snpEffVersionString = snpEffVersionLine.getValue().replaceAll("\"", "").split(" ")[0]; + + if ( ! isSupportedSnpEffVersion(snpEffVersionString) ) { + throw new UserException("The version of SnpEff used to generate the SnpEff input file (" + snpEffVersionString + ") " + + "is not currently supported by the GATK. Supported versions are: " + Arrays.toString(SUPPORTED_SNPEFF_VERSIONS)); + } + } + + private void checkSnpEffCommandLine ( VCFHeaderLine snpEffCommandLine ) { + if ( snpEffCommandLine == null || snpEffCommandLine.getValue() == null || snpEffCommandLine.getValue().trim().length() == 0 ) { + throw new UserException("Could not find a " + SNPEFF_VCF_HEADER_COMMAND_LINE_KEY + " entry in the VCF header for the SnpEff " + + "input file, which should be added by all supported versions of SnpEff (" + + Arrays.toString(SUPPORTED_SNPEFF_VERSIONS) + ")"); + } + } + + private boolean isSupportedSnpEffVersion ( String versionString ) { + for ( String supportedVersion : SUPPORTED_SNPEFF_VERSIONS ) { + if ( supportedVersion.equals(versionString) ) { + return true; + } + } + + return false; + } + + private VariantContext getMatchingSnpEffRecord ( List snpEffRecords, VariantContext vc ) { + for ( VariantContext snpEffRecord : snpEffRecords ) { + if ( snpEffRecord.hasSameAlternateAllelesAs(vc) && snpEffRecord.getReference().equals(vc.getReference()) ) { + return snpEffRecord; + } + } + + return null; + } + + private List parseSnpEffRecord ( VariantContext snpEffRecord ) { + List parsedEffects = new ArrayList(); + + Object effectFieldValue = snpEffRecord.getAttribute(SNPEFF_INFO_FIELD_KEY); + if ( effectFieldValue == null ) { + return parsedEffects; + } + + // The VCF codec stores multi-valued fields as a List, and single-valued fields as a String. + // We can have either in the case of SnpEff, since there may be one or more than one effect in this record. + List individualEffects; + if ( effectFieldValue instanceof List ) { + individualEffects = (List)effectFieldValue; + } + else { + individualEffects = Arrays.asList((String)effectFieldValue); + } + + for ( String effectString : individualEffects ) { + String[] effectNameAndMetadata = effectString.split(SNPEFF_EFFECT_METADATA_DELIMITER); + + if ( effectNameAndMetadata.length != 2 ) { + logger.warn(String.format("Malformed SnpEff effect field at %s:%d, skipping: %s", + snpEffRecord.getChr(), snpEffRecord.getStart(), effectString)); + continue; + } + + String effectName = effectNameAndMetadata[0]; + String[] effectMetadata = effectNameAndMetadata[1].split(SNPEFF_EFFECT_METADATA_SUBFIELD_DELIMITER, -1); + + SnpEffEffect parsedEffect = new SnpEffEffect(effectName, effectMetadata); + + if ( parsedEffect.isWellFormed() ) { + parsedEffects.add(parsedEffect); + } + else { + logger.warn(String.format("Skipping malformed SnpEff effect field at %s:%d. Error was: \"%s\". Field was: \"%s\"", + snpEffRecord.getChr(), snpEffRecord.getStart(), parsedEffect.getParseError(), effectString)); + } + } + + return parsedEffects; + } + + private SnpEffEffect getMostSignificantEffect ( List effects ) { + SnpEffEffect mostSignificantEffect = null; + + for ( SnpEffEffect effect : effects ) { if ( mostSignificantEffect == null || - snpEffFeature.isHigherImpactThan(mostSignificantEffect) ) { + effect.isHigherImpactThan(mostSignificantEffect) ) { - mostSignificantEffect = snpEffFeature; + mostSignificantEffect = effect; } } return mostSignificantEffect; } - private Map generateAnnotations ( SnpEffFeature mostSignificantEffect ) { - Map annotations = new LinkedHashMap(Utils.optimumHashSize(getKeyNames().size())); - - if ( mostSignificantEffect.hasGeneID() ) - annotations.put(GENE_ID_KEY, mostSignificantEffect.getGeneID()); - if ( mostSignificantEffect.hasGeneName() ) - annotations.put(GENE_NAME_KEY, mostSignificantEffect.getGeneName()); - if ( mostSignificantEffect.hasTranscriptID() ) - annotations.put(TRANSCRIPT_ID_KEY, mostSignificantEffect.getTranscriptID()); - if ( mostSignificantEffect.hasExonID() ) - annotations.put(EXON_ID_KEY, mostSignificantEffect.getExonID()); - if ( mostSignificantEffect.hasExonRank() ) - annotations.put(EXON_RANK_KEY, Integer.toString(mostSignificantEffect.getExonRank())); - if ( mostSignificantEffect.isNonCodingGene() ) - annotations.put(WITHIN_NON_CODING_GENE_KEY, null); - - annotations.put(EFFECT_KEY, mostSignificantEffect.getEffect().toString()); - annotations.put(EFFECT_IMPACT_KEY, mostSignificantEffect.getEffectImpact().toString()); - if ( mostSignificantEffect.hasEffectExtraInformation() ) - annotations.put(EFFECT_EXTRA_INFORMATION_KEY, mostSignificantEffect.getEffectExtraInformation()); - - if ( mostSignificantEffect.hasOldAndNewAA() ) - annotations.put(OLD_NEW_AA_KEY, mostSignificantEffect.getOldAndNewAA()); - if ( mostSignificantEffect.hasOldAndNewCodon() ) - annotations.put(OLD_NEW_CODON_KEY, mostSignificantEffect.getOldAndNewCodon()); - if ( mostSignificantEffect.hasCodonNum() ) - annotations.put(CODON_NUM_KEY, Integer.toString(mostSignificantEffect.getCodonNum())); - if ( mostSignificantEffect.hasCdsSize() ) - annotations.put(CDS_SIZE_KEY, Integer.toString(mostSignificantEffect.getCdsSize())); - - return annotations; - } - public List getKeyNames() { - return Arrays.asList( GENE_ID_KEY, - GENE_NAME_KEY, - TRANSCRIPT_ID_KEY, - EXON_ID_KEY, - EXON_RANK_KEY, - WITHIN_NON_CODING_GENE_KEY, - EFFECT_KEY, - EFFECT_IMPACT_KEY, - EFFECT_EXTRA_INFORMATION_KEY, - OLD_NEW_AA_KEY, - OLD_NEW_CODON_KEY, - CODON_NUM_KEY, - CDS_SIZE_KEY + return Arrays.asList( InfoFieldKey.EFFECT_KEY.getKeyName(), + InfoFieldKey.IMPACT_KEY.getKeyName(), + InfoFieldKey.CODON_CHANGE_KEY.getKeyName(), + InfoFieldKey.AMINO_ACID_CHANGE_KEY.getKeyName(), + InfoFieldKey.GENE_NAME_KEY.getKeyName(), + InfoFieldKey.GENE_BIOTYPE_KEY.getKeyName(), + InfoFieldKey.TRANSCRIPT_ID_KEY.getKeyName(), + InfoFieldKey.EXON_ID_KEY.getKeyName(), + InfoFieldKey.FUNCTIONAL_CLASS_KEY.getKeyName() ); } public List getDescriptions() { return Arrays.asList( - new VCFInfoHeaderLine(GENE_ID_KEY, 1, VCFHeaderLineType.String, "Gene ID for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(GENE_NAME_KEY, 1, VCFHeaderLineType.String, "Gene name for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(TRANSCRIPT_ID_KEY, 1, VCFHeaderLineType.String, "Transcript ID for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(EXON_ID_KEY, 1, VCFHeaderLineType.String, "Exon ID for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(EXON_RANK_KEY, 1, VCFHeaderLineType.Integer, "Exon rank for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(WITHIN_NON_CODING_GENE_KEY, 0, VCFHeaderLineType.Flag, "If this flag is present, the highest-impact effect resulting from the current variant is within a non-coding gene"), - new VCFInfoHeaderLine(EFFECT_KEY, 1, VCFHeaderLineType.String, "The highest-impact effect resulting from the current variant (or one of the highest-impact effects, if there is a tie)"), - new VCFInfoHeaderLine(EFFECT_IMPACT_KEY, 1, VCFHeaderLineType.String, "Impact of the highest-impact effect resulting from the current variant " + Arrays.toString(SnpEffConstants.EffectImpact.values())), - new VCFInfoHeaderLine(EFFECT_EXTRA_INFORMATION_KEY, 1, VCFHeaderLineType.String, "Additional information about the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(OLD_NEW_AA_KEY, 1, VCFHeaderLineType.String, "Old/New amino acid for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(OLD_NEW_CODON_KEY, 1, VCFHeaderLineType.String, "Old/New codon for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(CODON_NUM_KEY, 1, VCFHeaderLineType.Integer, "Codon number for the highest-impact effect resulting from the current variant"), - new VCFInfoHeaderLine(CDS_SIZE_KEY, 1, VCFHeaderLineType.Integer, "CDS size for the highest-impact effect resulting from the current variant") + new VCFInfoHeaderLine(InfoFieldKey.EFFECT_KEY.getKeyName(), 1, VCFHeaderLineType.String, "The highest-impact effect resulting from the current variant (or one of the highest-impact effects, if there is a tie)"), + new VCFInfoHeaderLine(InfoFieldKey.IMPACT_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Impact of the highest-impact effect resulting from the current variant " + Arrays.toString(EffectImpact.values())), + new VCFInfoHeaderLine(InfoFieldKey.CODON_CHANGE_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Old/New codon for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.AMINO_ACID_CHANGE_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Old/New amino acid for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.GENE_NAME_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Gene name for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.GENE_BIOTYPE_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Gene biotype for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.TRANSCRIPT_ID_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Transcript ID for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.EXON_ID_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Exon ID for the highest-impact effect resulting from the current variant"), + new VCFInfoHeaderLine(InfoFieldKey.FUNCTIONAL_CLASS_KEY.getKeyName(), 1, VCFHeaderLineType.String, "Functional class of the highest-impact effect resulting from the current variant: " + Arrays.toString(EffectFunctionalClass.values())) ); } + + /** + * Helper class to parse, validate, and store a single SnpEff effect and its metadata. + */ + protected static class SnpEffEffect { + private EffectType effect; + private EffectImpact impact; + private String codonChange; + private String aminoAcidChange; + private String geneName; + private String geneBiotype; + private EffectCoding coding; + private String transcriptID; + private String exonID; + + private String parseError = null; + private boolean isWellFormed = true; + + private static final int EXPECTED_NUMBER_OF_METADATA_FIELDS = 8; + private static final int NUMBER_OF_METADATA_FIELDS_UPON_WARNING = 9; + private static final int NUMBER_OF_METADATA_FIELDS_UPON_ERROR = 10; + + // Note that contrary to the description for the EFF field layout that SnpEff adds to the VCF header, + // errors come after warnings, not vice versa: + private static final int SNPEFF_WARNING_FIELD_INDEX = NUMBER_OF_METADATA_FIELDS_UPON_WARNING - 1; + private static final int SNPEFF_ERROR_FIELD_INDEX = NUMBER_OF_METADATA_FIELDS_UPON_ERROR - 1; + + private static final int SNPEFF_CODING_FIELD_INDEX = 5; + + public SnpEffEffect ( String effectName, String[] effectMetadata ) { + parseEffectName(effectName); + parseEffectMetadata(effectMetadata); + } + + private void parseEffectName ( String effectName ) { + try { + effect = EffectType.valueOf(effectName); + } + catch ( IllegalArgumentException e ) { + parseError(String.format("%s is not a recognized effect type", effectName)); + } + } + + private void parseEffectMetadata ( String[] effectMetadata ) { + if ( effectMetadata.length != EXPECTED_NUMBER_OF_METADATA_FIELDS ) { + if ( effectMetadata.length == NUMBER_OF_METADATA_FIELDS_UPON_WARNING ) { + parseError(String.format("SnpEff issued the following warning: %s", effectMetadata[SNPEFF_WARNING_FIELD_INDEX])); + } + else if ( effectMetadata.length == NUMBER_OF_METADATA_FIELDS_UPON_ERROR ) { + parseError(String.format("SnpEff issued the following error: %s", effectMetadata[SNPEFF_ERROR_FIELD_INDEX])); + } + else { + parseError(String.format("Wrong number of effect metadata fields. Expected %d but found %d", + EXPECTED_NUMBER_OF_METADATA_FIELDS, effectMetadata.length)); + } + + return; + } + + if ( effect != null && effect.isModifier() ) { + impact = EffectImpact.MODIFIER; + } + else { + try { + impact = EffectImpact.valueOf(effectMetadata[InfoFieldKey.IMPACT_KEY.getFieldIndex()]); + } + catch ( IllegalArgumentException e ) { + parseError(String.format("Unrecognized value for effect impact: %s", effectMetadata[InfoFieldKey.IMPACT_KEY.getFieldIndex()])); + } + } + + codonChange = effectMetadata[InfoFieldKey.CODON_CHANGE_KEY.getFieldIndex()]; + aminoAcidChange = effectMetadata[InfoFieldKey.AMINO_ACID_CHANGE_KEY.getFieldIndex()]; + geneName = effectMetadata[InfoFieldKey.GENE_NAME_KEY.getFieldIndex()]; + geneBiotype = effectMetadata[InfoFieldKey.GENE_BIOTYPE_KEY.getFieldIndex()]; + + if ( effectMetadata[SNPEFF_CODING_FIELD_INDEX].trim().length() > 0 ) { + try { + coding = EffectCoding.valueOf(effectMetadata[SNPEFF_CODING_FIELD_INDEX]); + } + catch ( IllegalArgumentException e ) { + parseError(String.format("Unrecognized value for effect coding: %s", effectMetadata[SNPEFF_CODING_FIELD_INDEX])); + } + } + else { + coding = EffectCoding.UNKNOWN; + } + + transcriptID = effectMetadata[InfoFieldKey.TRANSCRIPT_ID_KEY.getFieldIndex()]; + exonID = effectMetadata[InfoFieldKey.EXON_ID_KEY.getFieldIndex()]; + } + + private void parseError ( String message ) { + isWellFormed = false; + + // Cache only the first error encountered: + if ( parseError == null ) { + parseError = message; + } + } + + public boolean isWellFormed() { + return isWellFormed; + } + + public String getParseError() { + return parseError == null ? "" : parseError; + } + + public boolean isCoding() { + return coding == EffectCoding.CODING; + } + + public boolean isHigherImpactThan ( SnpEffEffect other ) { + // If one effect is within a coding gene and the other is not, the effect that is + // within the coding gene has higher impact: + + if ( isCoding() && ! other.isCoding() ) { + return true; + } + else if ( ! isCoding() && other.isCoding() ) { + return false; + } + + // Otherwise, both effects are either in or not in a coding gene, so we compare the impacts + // of the effects themselves. Effects with the same impact are tie-broken using the + // functional class of the effect: + + if ( impact.isHigherImpactThan(other.impact) ) { + return true; + } + else if ( impact.isSameImpactAs(other.impact) ) { + return effect.getFunctionalClass().isHigherPriorityThan(other.effect.getFunctionalClass()); + } + + return false; + } + + public Map getAnnotations() { + Map annotations = new LinkedHashMap(Utils.optimumHashSize(InfoFieldKey.values().length)); + + addAnnotation(annotations, InfoFieldKey.EFFECT_KEY.getKeyName(), effect.toString()); + addAnnotation(annotations, InfoFieldKey.IMPACT_KEY.getKeyName(), impact.toString()); + addAnnotation(annotations, InfoFieldKey.CODON_CHANGE_KEY.getKeyName(), codonChange); + addAnnotation(annotations, InfoFieldKey.AMINO_ACID_CHANGE_KEY.getKeyName(), aminoAcidChange); + addAnnotation(annotations, InfoFieldKey.GENE_NAME_KEY.getKeyName(), geneName); + addAnnotation(annotations, InfoFieldKey.GENE_BIOTYPE_KEY.getKeyName(), geneBiotype); + addAnnotation(annotations, InfoFieldKey.TRANSCRIPT_ID_KEY.getKeyName(), transcriptID); + addAnnotation(annotations, InfoFieldKey.EXON_ID_KEY.getKeyName(), exonID); + addAnnotation(annotations, InfoFieldKey.FUNCTIONAL_CLASS_KEY.getKeyName(), effect.getFunctionalClass().toString()); + + return annotations; + } + + private void addAnnotation ( Map annotations, String keyName, String keyValue ) { + // Only add annotations for keys associated with non-empty values: + if ( keyValue != null && keyValue.trim().length() > 0 ) { + annotations.put(keyName, keyValue); + } + } + } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java index 42203824f..f747fbc2e 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/SpanningDeletions.java @@ -17,6 +17,9 @@ import java.util.List; import java.util.Map; +/** + * Fraction of reads containing spanning deletions at this site. + */ public class SpanningDeletions extends InfoFieldAnnotation implements StandardAnnotation { public Map annotate(RefMetaDataTracker tracker, AnnotatorCompatibleWalker walker, ReferenceContext ref, Map stratifiedContexts, VariantContext vc) { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java index fa48c57a3..1f5508f4c 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/TechnologyComposition.java @@ -1,5 +1,6 @@ package org.broadinstitute.sting.gatk.walkers.annotator; +import org.broadinstitute.sting.commandline.Hidden; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; @@ -19,12 +20,9 @@ import java.util.List; import java.util.Map; /** - * Created by IntelliJ IDEA. - * User: delangel - * Date: 6/29/11 - * Time: 3:14 PM - * To change this template use File | Settings | File Templates. + * Counts of bases from SLX, 454, and SOLiD at this site */ +@Hidden public class TechnologyComposition extends InfoFieldAnnotation implements ExperimentalAnnotation { private String nSLX = "NumSLX"; private String n454 ="Num454"; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java index 96a400c68..3be87da80 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotator.java @@ -40,7 +40,6 @@ import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnot import org.broadinstitute.sting.utils.BaseUtils; import org.broadinstitute.sting.utils.SampleUtils; import org.broadinstitute.sting.utils.classloader.PluginManager; -import org.broadinstitute.sting.utils.codecs.snpEff.SnpEffFeature; import org.broadinstitute.sting.utils.codecs.vcf.*; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import org.broadinstitute.sting.utils.variantcontext.VariantContextUtils; @@ -86,14 +85,15 @@ public class VariantAnnotator extends RodWalker implements Ann @ArgumentCollection protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection(); + public RodBinding getVariantRodBinding() { return variantCollection.variants; } /** * The INFO field will be annotated with information on the most biologically-significant effect * listed in the SnpEff output file for each variant. */ @Input(fullName="snpEffFile", shortName = "snpEffFile", doc="A SnpEff output file from which to add annotations", required=false) - public RodBinding snpEffFile; - public RodBinding getSnpEffRodBinding() { return snpEffFile; } + public RodBinding snpEffFile; + public RodBinding getSnpEffRodBinding() { return snpEffFile; } /** * rsIDs from this file are used to populate the ID column of the output. Also, the DB INFO flag will be set when appropriate. @@ -162,6 +162,12 @@ public class VariantAnnotator extends RodWalker implements Ann @Argument(fullName="vcfContainsOnlyIndels", shortName="dels",doc="Use if you are annotating an indel vcf, currently VERY experimental", required = false) protected boolean indelsOnly = false; + @Argument(fullName="family_string",shortName="family",required=false,doc="A family string of the form mom+dad=child for use with the mendelian violation ratio annotation") + public String familyStr = null; + + @Argument(fullName="MendelViolationGenotypeQualityThreshold",shortName="mvq",required=false,doc="The genotype quality treshold in order to annotate mendelian violation ratio") + public double minGenotypeQualityP = 0.0; + private VariantAnnotatorEngine engine; private Collection indelBufferContext; @@ -203,9 +209,9 @@ public class VariantAnnotator extends RodWalker implements Ann } if ( USE_ALL_ANNOTATIONS ) - engine = new VariantAnnotatorEngine(this); + engine = new VariantAnnotatorEngine(this, getToolkit()); else - engine = new VariantAnnotatorEngine(annotationGroupsToUse, annotationsToUse, this); + engine = new VariantAnnotatorEngine(annotationGroupsToUse, annotationsToUse, this, getToolkit()); engine.initializeExpressions(expressionsToUse); // setup the header fields @@ -217,6 +223,8 @@ public class VariantAnnotator extends RodWalker implements Ann hInfo.add(line); } + engine.invokeAnnotationInitializationMethods(hInfo); + VCFHeader vcfHeader = new VCFHeader(hInfo, samples); vcfWriter.writeHeader(vcfHeader); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java index 01926a7f3..e5effe6d8 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorEngine.java @@ -26,13 +26,11 @@ package org.broadinstitute.sting.gatk.walkers.annotator; import org.broadinstitute.sting.commandline.RodBinding; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotationInterfaceManager; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.GenotypeAnnotation; -import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.InfoFieldAnnotation; +import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.*; import org.broadinstitute.sting.utils.codecs.vcf.*; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.variantcontext.Genotype; @@ -49,6 +47,7 @@ public class VariantAnnotatorEngine { private HashMap, String> dbAnnotations = new HashMap, String>(); private AnnotatorCompatibleWalker walker; + private GenomeAnalysisEngine toolkit; private static class VAExpression { @@ -74,16 +73,18 @@ public class VariantAnnotatorEngine { } // use this constructor if you want all possible annotations - public VariantAnnotatorEngine(AnnotatorCompatibleWalker walker) { + public VariantAnnotatorEngine(AnnotatorCompatibleWalker walker, GenomeAnalysisEngine toolkit) { this.walker = walker; + this.toolkit = toolkit; requestedInfoAnnotations = AnnotationInterfaceManager.createAllInfoFieldAnnotations(); requestedGenotypeAnnotations = AnnotationInterfaceManager.createAllGenotypeAnnotations(); initializeDBs(); } // use this constructor if you want to select specific annotations (and/or interfaces) - public VariantAnnotatorEngine(List annotationGroupsToUse, List annotationsToUse, AnnotatorCompatibleWalker walker) { + public VariantAnnotatorEngine(List annotationGroupsToUse, List annotationsToUse, AnnotatorCompatibleWalker walker, GenomeAnalysisEngine toolkit) { this.walker = walker; + this.toolkit = toolkit; initializeAnnotations(annotationGroupsToUse, annotationsToUse); initializeDBs(); } @@ -113,6 +114,16 @@ public class VariantAnnotatorEngine { dbAnnotations.put(rod, rod.getName()); } + public void invokeAnnotationInitializationMethods( Set headerLines ) { + for ( VariantAnnotatorAnnotation annotation : requestedInfoAnnotations ) { + annotation.initialize(walker, toolkit, headerLines); + } + + for ( VariantAnnotatorAnnotation annotation : requestedGenotypeAnnotations ) { + annotation.initialize(walker, toolkit, headerLines); + } + } + public Set getVCFAnnotationDescriptions() { Set descriptions = new HashSet(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java index 20a2aea0e..7200f841b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/AnnotatorCompatibleWalker.java @@ -1,7 +1,6 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces; import org.broadinstitute.sting.commandline.RodBinding; -import org.broadinstitute.sting.utils.codecs.snpEff.SnpEffFeature; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.List; @@ -9,8 +8,9 @@ import java.util.List; public interface AnnotatorCompatibleWalker { // getter methods for various used bindings - public abstract RodBinding getSnpEffRodBinding(); + public abstract RodBinding getVariantRodBinding(); + public abstract RodBinding getSnpEffRodBinding(); public abstract RodBinding getDbsnpRodBinding(); public abstract List> getCompRodBindings(); public abstract List> getResourceRodBindings(); -} \ No newline at end of file +} diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/VariantAnnotatorAnnotation.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/VariantAnnotatorAnnotation.java index f33d61df9..521f89016 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/VariantAnnotatorAnnotation.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/annotator/interfaces/VariantAnnotatorAnnotation.java @@ -24,18 +24,18 @@ package org.broadinstitute.sting.gatk.walkers.annotator.interfaces; -import org.broadinstitute.sting.gatk.contexts.AlignmentContext; -import org.broadinstitute.sting.gatk.contexts.ReferenceContext; -import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker; -import org.broadinstitute.sting.utils.codecs.vcf.VCFInfoHeaderLine; +import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.utils.codecs.vcf.VCFHeaderLine; import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; -import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.List; -import java.util.Map; +import java.util.Set; @DocumentedGATKFeature(enable = true, groupName = "VariantAnnotator annotations", summary = "VariantAnnotator annotations") public abstract class VariantAnnotatorAnnotation { // return the INFO keys public abstract List getKeyNames(); + + // initialization method (optional for subclasses, and therefore non-abstract) + public void initialize ( AnnotatorCompatibleWalker walker, GenomeAnalysisEngine toolkit, Set headerLines ) { } } \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java index 880dba5d0..7f6dabeec 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/beagle/BeagleOutputToVCFWalker.java @@ -175,21 +175,16 @@ public class BeagleOutputToVCFWalker extends RodWalker { } BeagleFeature beagleR2Feature = tracker.getFirstValue(beagleR2); - // ignore places where we don't have a variant - if ( beagleR2Feature == null ) - return 0; - - BeagleFeature beagleProbsFeature = tracker.getFirstValue(beagleProbs); - - // ignore places where we don't have a variant - if ( beagleProbsFeature == null ) - return 0; - BeagleFeature beaglePhasedFeature = tracker.getFirstValue(beaglePhased); + // ignore places where we don't have a variant - if ( beaglePhasedFeature == null ) - return 0; + if ( beagleR2Feature == null || beagleProbsFeature == null || beaglePhasedFeature == null) + { + vcfWriter.add(vc_input); + return 1; + } + // get reference base for current position byte refByte = ref.getBase(); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java index 3a18fe610..664c319ab 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/coverage/DepthOfCoverageWalker.java @@ -63,20 +63,32 @@ import java.util.*; *

Input

*

* One or more bam files (with proper headers) to be analyzed for coverage statistics - * (Optional) A REFSEQ Rod to aggregate coverage to the gene level *

- * + *

+ *(Optional) A REFSEQ Rod to aggregate coverage to the gene level + *

+ * (for information about creating the REFSEQ Rod, please consult the RefSeqCodec documentation) + *

*

Output

*

* Tables pertaining to different coverage summaries. Suffix on the table files declares the contents: + *

* - no suffix: per locus coverage + *

* - _summary: total, mean, median, quartiles, and threshold proportions, aggregated over all bases + *

* - _statistics: coverage histograms (# locus with X coverage), aggregated over all bases + *

* - _interval_summary: total, mean, median, quartiles, and threshold proportions, aggregated per interval + *

* - _interval_statistics: 2x2 table of # of intervals covered to >= X depth in >=Y samples + *

* - _gene_summary: total, mean, median, quartiles, and threshold proportions, aggregated per gene + *

* - _gene_statistics: 2x2 table of # of genes covered to >= X depth in >= Y samples + *

* - _cumulative_coverage_counts: coverage histograms (# locus with >= X coverage), aggregated over all bases + *

* - _cumulative_coverage_proportions: proprotions of loci with >= X coverage, aggregated over all bases *

* @@ -84,7 +96,7 @@ import java.util.*; * 
  * java -Xmx2g -jar GenomeAnalysisTK.jar \
  *   -R ref.fasta \
- *   -T VariantEval \
+ *   -T DepthOfCoverage \
  *   -o file_name_base \
  *   -I input_bams.list
  *   [-geneList refSeq.sorted.txt] \
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaAlternateReferenceWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaAlternateReferenceWalker.java
index fd912334f..4e2c17bf6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaAlternateReferenceWalker.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaAlternateReferenceWalker.java
@@ -43,8 +43,10 @@ import java.util.List;
  * Generates an alternative reference sequence over the specified interval.
  *
  * 

- * Given variant ROD tracks, it replaces the reference bases at variation sites with the bases supplied by the ROD(s).
- * Additionally, allows for a "snpmask" ROD to set overlapping bases to 'N'.
+ * Given variant tracks, it replaces the reference bases at variation sites with the bases supplied by the ROD(s).
+ * Additionally, allows for one or more "snpmask" VCFs to set overlapping bases to 'N'.
+ * Note that if there are multiple variants at a site, it takes the first one seen.
+ * Reference bases for each interval will be output as a separate fasta sequence (named numerically in order).
  *
  * 
Input

  * 

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaReferenceWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaReferenceWalker.java
index 5f3b37cc8..7ae5c5c75 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaReferenceWalker.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/fasta/FastaReferenceWalker.java
@@ -42,6 +42,9 @@ import java.io.PrintStream;
  *
  * 

  * The output format can be partially controlled using the provided command-line arguments.
+ * Specify intervals with the usual -L argument to output only the reference bases within your intervals.
+ * Overlapping intervals are automatically merged; reference bases for each disjoint interval will be output as a
+ * separate fasta sequence (named numerically in order).
  *
  * 
Input

  * 

diff --git a/public/java/src/org/broadinstitute/sting/utils/genotype/DiploidGenotype.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotype.java
similarity index 98%
rename from public/java/src/org/broadinstitute/sting/utils/genotype/DiploidGenotype.java
rename to public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotype.java
index 1c2cfe2e1..b5987963f 100755
--- a/public/java/src/org/broadinstitute/sting/utils/genotype/DiploidGenotype.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotype.java
@@ -23,7 +23,7 @@
  * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package org.broadinstitute.sting.utils.genotype;
+package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import org.broadinstitute.sting.utils.BaseUtils;
 
@@ -34,7 +34,7 @@ import org.broadinstitute.sting.utils.BaseUtils;
  * Time: 6:46:09 PM
  * To change this template use File | Settings | File Templates.
  */
-public enum DiploidGenotype {
+enum DiploidGenotype {
     AA ('A', 'A'),
     AC ('A', 'C'),
     AG ('A', 'G'),
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidIndelGenotypePriors.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidIndelGenotypePriors.java
index 696a74de8..d8c911092 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidIndelGenotypePriors.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidIndelGenotypePriors.java
@@ -2,7 +2,6 @@ package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import org.broadinstitute.sting.gatk.walkers.indels.HaplotypeIndelErrorModel;
 import org.broadinstitute.sting.utils.MathUtils;
-import org.broadinstitute.sting.utils.genotype.DiploidGenotype;
 
 /**
  * Created by IntelliJ IDEA.
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java
index 5f6865d04..71eea2467 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypeLikelihoods.java
@@ -30,7 +30,6 @@ import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.QualityUtils;
 import org.broadinstitute.sting.utils.exceptions.UserException;
-import org.broadinstitute.sting.utils.genotype.DiploidGenotype;
 import org.broadinstitute.sting.utils.pileup.FragmentPileup;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
@@ -276,8 +275,11 @@ public class DiploidSNPGenotypeLikelihoods implements Cloneable {
         if ( elt.isReducedRead() ) {
             // reduced read representation
             byte qual = elt.getReducedQual();
-            add(obsBase, qual, (byte)0, (byte)0, elt.getReducedCount()); // fast calculation of n identical likelihoods
-            return elt.getReducedCount(); // we added nObs bases here
+            if ( BaseUtils.isRegularBase( elt.getBase() )) {
+                add(obsBase, qual, (byte)0, (byte)0, elt.getReducedCount()); // fast calculation of n identical likelihoods
+                return elt.getReducedCount(); // we added nObs bases here
+            } else // odd bases or deletions => don't use them
+                return 0;
         } else {
             byte qual = qualToUse(elt, ignoreBadBases, capBaseQualsAtMappingQual, minBaseQual);
             return qual > 0 ? add(obsBase, qual, (byte)0, (byte)0, 1) : 0;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypePriors.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypePriors.java
index b9ed17d3e..71854591f 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypePriors.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidSNPGenotypePriors.java
@@ -26,7 +26,6 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import org.broadinstitute.sting.utils.MathUtils;
-import org.broadinstitute.sting.utils.genotype.DiploidGenotype;
 
 import java.util.Arrays;
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java
index cd006a3cf..8a3a97823 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/ExactAFCalculationModel.java
@@ -48,27 +48,12 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
     // code for testing purposes
     //
     private final static boolean DEBUG = false;
-    private final static boolean PRINT_LIKELIHOODS = false;
-    private final static int N_CYCLES = 1;
-    private SimpleTimer timerExpt = new SimpleTimer("linearExactBanded");
-    private SimpleTimer timerGS = new SimpleTimer("linearExactGS");
-    private final static boolean COMPARE_TO_GS = false;
-
-    public enum ExactCalculation {
-        N2_GOLD_STANDARD,
-        LINEAR_EXPERIMENTAL
-    }
-
     private final static double MAX_LOG10_ERROR_TO_STOP_EARLY = 6; // we want the calculation to be accurate to 1 / 10^6
+    private final boolean SIMPLE_GREEDY_GENOTYPER = false;
+    private final static double SUM_GL_THRESH_NOCALL = -0.001; // if sum(gl) is bigger than this threshold, we treat GL's as non-informative and will force a no-call.
 
-    private boolean SIMPLE_GREEDY_GENOTYPER = false;
-
-
-
-    final private ExactCalculation calcToUse;
     protected ExactAFCalculationModel(UnifiedArgumentCollection UAC, int N, Logger logger, PrintStream verboseWriter) {
         super(UAC, N, logger, verboseWriter);
-        calcToUse = UAC.EXACT_CALCULATION_TYPE;
     }
 
     public void getLog10PNonRef(RefMetaDataTracker tracker,
@@ -76,43 +61,12 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
                                 Map GLs, Setalleles,
                                 double[] log10AlleleFrequencyPriors,
                                 double[] log10AlleleFrequencyPosteriors) {
-        // todo -- REMOVE ME AFTER TESTING
-        // todo -- REMOVE ME AFTER TESTING
-        // todo -- REMOVE ME AFTER TESTING
-        double[] gsPosteriors;
-        if ( COMPARE_TO_GS ) // due to annoying special values in incoming array, we have to clone up here
-            gsPosteriors = log10AlleleFrequencyPosteriors.clone();
-
-        int idxAA = GenotypeType.AA.ordinal();
-        int idxAB = GenotypeType.AB.ordinal();
-        int idxBB = GenotypeType.BB.ordinal();
-
-        // todo -- remove me after testing
-        if ( N_CYCLES > 1 ) {
-            for ( int i = 0; i < N_CYCLES; i++) {
-                timerGS.restart();
-                linearExact(GLs, log10AlleleFrequencyPriors, log10AlleleFrequencyPosteriors.clone(), idxAA, idxAB, idxBB);
-                timerGS.stop();
-
-                timerExpt.restart();
-                linearExactBanded(GLs, log10AlleleFrequencyPriors, log10AlleleFrequencyPosteriors.clone());
-                timerExpt.stop();
-            }
-
-            System.out.printf("good = %.2f, expt = %.2f, delta = %.2f%n",
-                    timerGS.getElapsedTime(), timerExpt.getElapsedTime(), timerExpt.getElapsedTime()-timerGS.getElapsedTime());
-        }
-
-        int lastK = -1;
-
-        int numAlleles = alleles.size();
+        final int numAlleles = alleles.size();
+        final double[][] posteriorCache = numAlleles > 2 ? new double[numAlleles-1][] : null;
+        final double[] bestAFguess = numAlleles > 2 ? new double[numAlleles-1] : null;
 
         int idxDiag = numAlleles;
         int incr = numAlleles - 1;
-
-        double[][] posteriorCache = new double[numAlleles-1][];
-        double[] bestAFguess = new double[numAlleles-1];
-
         for (int k=1; k < numAlleles; k++) {
             // multi-allelic approximation, part 1: Ideally
             // for each alt allele compute marginal (suboptimal) posteriors -
@@ -121,24 +75,17 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
             // So, for example, with 2 alt alleles, likelihoods have AA,AB,AC,BB,BC,CC.
             // 3 alt alleles: AA,AB,AC,AD BB BC BD CC CD DD
 
-            idxAA = 0;
-            idxAB = k;
+            final int idxAA = 0;
+            final int idxAB = k;
             // yy is always element on the diagonal.
             // 2 alleles: BBelement 2
             // 3 alleles: BB element  3. CC element 5
             // 4 alleles:
-            idxBB = idxDiag;
+            final int idxBB = idxDiag;
             idxDiag += incr--;
 
-            // todo - possible cleanup
-            switch ( calcToUse ) {
-                case N2_GOLD_STANDARD:
-                    lastK = gdaN2GoldStandard(GLs, log10AlleleFrequencyPriors, log10AlleleFrequencyPosteriors, idxAA, idxAB, idxBB);
-                    break;
-                case LINEAR_EXPERIMENTAL:
-                    lastK = linearExact(GLs, log10AlleleFrequencyPriors, log10AlleleFrequencyPosteriors, idxAA, idxAB, idxBB);
-                    break;
-            }
+            final int lastK = linearExact(GLs, log10AlleleFrequencyPriors, log10AlleleFrequencyPosteriors, idxAA, idxAB, idxBB);
+
             if (numAlleles > 2) {
                 posteriorCache[k-1] = log10AlleleFrequencyPosteriors.clone();
                 bestAFguess[k-1] = (double)MathUtils.maxElementIndex(log10AlleleFrequencyPosteriors);
@@ -153,47 +100,25 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
                 log10AlleleFrequencyPosteriors[k] = (posteriorCache[mostLikelyAlleleIdx][k]);
 
         }
-        // todo -- REMOVE ME AFTER TESTING
-        // todo -- REMOVE ME AFTER TESTING
-        // todo -- REMOVE ME AFTER TESTING
-        if ( COMPARE_TO_GS ) {
-            gdaN2GoldStandard(GLs, log10AlleleFrequencyPriors, gsPosteriors, idxAA, idxAB, idxBB);
-
-            double log10thisPVar = Math.log10(MathUtils.normalizeFromLog10(log10AlleleFrequencyPosteriors)[0]);
-            double log10gsPVar = Math.log10(MathUtils.normalizeFromLog10(gsPosteriors)[0]);
-            boolean eq = (log10thisPVar == Double.NEGATIVE_INFINITY && log10gsPVar == Double.NEGATIVE_INFINITY) || MathUtils.compareDoubles(log10thisPVar, log10gsPVar, 1e-4) == 0;
-
-            if ( ! eq || PRINT_LIKELIHOODS ) {
-                System.out.printf("----------------------------------------%n");
-                for (int k=0; k < log10AlleleFrequencyPosteriors.length; k++) {
-                    double x = log10AlleleFrequencyPosteriors[k];
-                    System.out.printf("  %d\t%.2f\t%.2f\t%b%n", k,
-                            x < -1e10 ? Double.NEGATIVE_INFINITY : x, gsPosteriors[k],
-                            log10AlleleFrequencyPosteriors[k] == gsPosteriors[k]);
-                }
-                System.out.printf("MAD_AC\t%d\t%d\t%.2f\t%.2f\t%.6f%n",
-                        ref.getLocus().getStart(), lastK, log10thisPVar, log10gsPVar, log10thisPVar - log10gsPVar);
-            }
-        }
-
     }
 
-    private static final double[][] getGLs(Map GLs) {
-        double[][] genotypeLikelihoods = new double[GLs.size()+1][];
+    private static final ArrayList getGLs(Map GLs) {
+        ArrayList genotypeLikelihoods = new ArrayList();
 
-        int j = 0;
+        genotypeLikelihoods.add(new double[]{0.0,0.0,0.0}); // dummy
         for ( Genotype sample : GLs.values() ) {
-            j++;
-
             if ( sample.hasLikelihoods() ) {
-                //double[] genotypeLikelihoods = MathUtils.normalizeFromLog10(GLs.get(sample).getLikelihoods());
-                genotypeLikelihoods[j] = sample.getLikelihoods().getAsVector();
+                double[] gls = sample.getLikelihoods().getAsVector();
+
+                if (MathUtils.sum(gls) < SUM_GL_THRESH_NOCALL)
+                    genotypeLikelihoods.add(gls);
             }
         }
 
         return genotypeLikelihoods;
     }
 
+
     // -------------------------------------------------------------------------------------
     //
     // Linearized, ~O(N), implementation.
@@ -237,90 +162,12 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
         }
     }
 
-    // now with banding
-    public int linearExactBanded(Map GLs,
-                                 double[] log10AlleleFrequencyPriors,
-                                 double[] log10AlleleFrequencyPosteriors) {
-        throw new NotImplementedException();
-//        final int numSamples = GLs.size();
-//        final int numChr = 2*numSamples;
-//        final double[][] genotypeLikelihoods = getGLs(GLs);
-//
-//        final ExactACCache logY = new ExactACCache(numSamples+1);
-//        logY.getkMinus0()[0] = 0.0; // the zero case
-//
-//        double maxLog10L = Double.NEGATIVE_INFINITY;
-//        boolean done = false;
-//        int lastK = -1;
-//        final int BAND_SIZE = 10;
-//
-//        for (int k=0; k <= numChr && ! done; k++ ) {
-//            final double[] kMinus0 = logY.getkMinus0();
-//            int jStart = Math.max(k - BAND_SIZE, 1);
-//            int jStop = Math.min(k + BAND_SIZE, numSamples);
-//
-//            if ( k == 0 ) { // special case for k = 0
-//                for ( int j=1; j <= numSamples; j++ ) {
-//                    kMinus0[j] = kMinus0[j-1] + genotypeLikelihoods[j][GenotypeType.AA.ordinal()];
-//                }
-//            } else { // k > 0
-//                final double[] kMinus1 = logY.getkMinus1();
-//                final double[] kMinus2 = logY.getkMinus2();
-//                Arrays.fill(kMinus0,0);
-//
-//                for ( int j = jStart; j <= jStop; j++ ) {
-//                    final double[] gl = genotypeLikelihoods[j];
-//                    final double logDenominator = log10Cache[2*j] + log10Cache[2*j-1];
-//
-//                    double aa = Double.NEGATIVE_INFINITY;
-//                    double ab = Double.NEGATIVE_INFINITY;
-//                    if (k < 2*j-1)
-//                        aa = log10Cache[2*j-k] + log10Cache[2*j-k-1] + kMinus0[j-1] + gl[GenotypeType.AA.ordinal()];
-//
-//                    if (k < 2*j)
-//                        ab = log10Cache[2*k] + log10Cache[2*j-k]+ kMinus1[j-1] + gl[GenotypeType.AB.ordinal()];
-//
-//                    double log10Max;
-//                    if (k > 1) {
-//                        final double bb = log10Cache[k] + log10Cache[k-1] + kMinus2[j-1] + gl[GenotypeType.BB.ordinal()];
-//                        log10Max = approximateLog10SumLog10(aa, ab, bb);
-//                    } else {
-//                        // we know we aren't considering the BB case, so we can use an optimized log10 function
-//                        log10Max = approximateLog10SumLog10(aa, ab);
-//                    }
-//
-//                    // finally, update the L(j,k) value
-//                    kMinus0[j] = log10Max - logDenominator;
-//
-//                    String offset = Utils.dupString(' ',k);
-//                    System.out.printf("%s%3d %3d %.2f%n", offset, k, j, kMinus0[j]);
-//                }
-//            }
-//
-//            // update the posteriors vector
-//            final double log10LofK = kMinus0[jStop];
-//            log10AlleleFrequencyPosteriors[k] = log10LofK + log10AlleleFrequencyPriors[k];
-//
-//            // can we abort early?
-//            lastK = k;
-//            maxLog10L = Math.max(maxLog10L, log10LofK);
-//            if ( log10LofK < maxLog10L - MAX_LOG10_ERROR_TO_STOP_EARLY ) {
-//                if ( DEBUG ) System.out.printf("  *** breaking early k=%d log10L=%.2f maxLog10L=%.2f%n", k, log10LofK, maxLog10L);
-//                done = true;
-//            }
-//
-//            logY.rotate();
-//        }
-//
-//        return lastK;
-    }
-
     public int linearExact(Map GLs,
                            double[] log10AlleleFrequencyPriors,
                            double[] log10AlleleFrequencyPosteriors, int idxAA, int idxAB, int idxBB) {
-        final int numSamples = GLs.size();
+        final ArrayList genotypeLikelihoods = getGLs(GLs);
+        final int numSamples = genotypeLikelihoods.size()-1;
         final int numChr = 2*numSamples;
-        final double[][] genotypeLikelihoods = getGLs(GLs);
 
         final ExactACCache logY = new ExactACCache(numSamples+1);
         logY.getkMinus0()[0] = 0.0; // the zero case
@@ -334,14 +181,14 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
 
             if ( k == 0 ) { // special case for k = 0
                 for ( int j=1; j <= numSamples; j++ ) {
-                    kMinus0[j] = kMinus0[j-1] + genotypeLikelihoods[j][idxAA];
+                    kMinus0[j] = kMinus0[j-1] + genotypeLikelihoods.get(j)[idxAA];
                 }
             } else { // k > 0
                 final double[] kMinus1 = logY.getkMinus1();
                 final double[] kMinus2 = logY.getkMinus2();
 
                 for ( int j=1; j <= numSamples; j++ ) {
-                    final double[] gl = genotypeLikelihoods[j];
+                    final double[] gl = genotypeLikelihoods.get(j);
                     final double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1];
 
                     double aa = Double.NEGATIVE_INFINITY;
@@ -434,10 +281,6 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
         if ( !vc.isVariant() )
             throw new UserException("The VCF record passed in does not contain an ALT allele at " + vc.getChr() + ":" + vc.getStart());
 
-        boolean multiAllelicRecord = false;
-
-        if (vc.getAlternateAlleles().size() > 1)
-            multiAllelicRecord = true;
 
         Map GLs = vc.getGenotypes();
         double[][] pathMetricArray = new double[GLs.size()+1][AFofMaxLikelihood+1];
@@ -454,7 +297,7 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
         pathMetricArray[0][0] = 0.0;
 
         // todo = can't deal with optimal dynamic programming solution with multiallelic records
-        if (SIMPLE_GREEDY_GENOTYPER || multiAllelicRecord) {
+        if (SIMPLE_GREEDY_GENOTYPER || !vc.isBiallelic()) {
             sampleIndices.addAll(GLs.keySet());
             sampleIdx = GLs.size();
         }
@@ -465,6 +308,17 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
                     continue;
 
                 double[] likelihoods = sample.getValue().getLikelihoods().getAsVector();
+
+                if (MathUtils.sum(likelihoods) > SUM_GL_THRESH_NOCALL)     {
+                    //System.out.print(sample.getKey()+":");
+                    //for (int k=0; k < likelihoods.length; k++)
+                    //   System.out.format("%4.2f ",likelihoods[k]);
+                    //System.out.println();
+                    // all likelihoods are essentially the same: skip this sample and will later on force no call.
+                    //sampleIdx++;
+                    continue;
+                }
+
                 sampleIndices.add(sample.getKey());
 
                 for (int k=0; k <= AFofMaxLikelihood; k++) {
@@ -504,22 +358,25 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
             Genotype g = GLs.get(sample);
             if ( !g.hasLikelihoods() )
                 continue;
-
-            if (SIMPLE_GREEDY_GENOTYPER || multiAllelicRecord)
-                bestGTguess = Utils.findIndexOfMaxEntry(g.getLikelihoods().getAsVector());
-            else {
-                int newIdx = tracebackArray[k][startIdx];
-                bestGTguess = startIdx - newIdx;
-                startIdx = newIdx;
-            }
-
+            // if all likelihoods are essentially the same: we want to force no-call. In this case, we skip this sample for now,
+            // and will add no-call genotype to GL's in a second pass
             ArrayList myAlleles = new ArrayList();
 
             double qual = Double.NEGATIVE_INFINITY;
             double[] likelihoods = g.getLikelihoods().getAsVector();
+
+            if (SIMPLE_GREEDY_GENOTYPER || !vc.isBiallelic()) {
+                bestGTguess = Utils.findIndexOfMaxEntry(g.getLikelihoods().getAsVector());
+            }
+            else {
+                int newIdx = tracebackArray[k][startIdx];;
+                bestGTguess = startIdx - newIdx;
+                startIdx = newIdx;
+            }
+
             /*           System.out.format("Sample: %s GL:",sample);
                     for (int i=0; i < likelihoods.length; i++)
-                        System.out.format("%1.4f ",likelihoods[i]);
+                        System.out.format("%1.4f, ",likelihoods[i]);
             */
 
             for (int i=0; i < likelihoods.length; i++) {
@@ -570,83 +427,26 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
 
         }
 
-        return calls;
-    }
-
-    // -------------------------------------------------------------------------------------
-    //
-    // Gold standard, but O(N^2), implementation.
-    //
-    // TODO -- remove me for clarity in this code
-    //
-    // -------------------------------------------------------------------------------------
-    public int gdaN2GoldStandard(Map GLs,
-                                 double[] log10AlleleFrequencyPriors,
-                                 double[] log10AlleleFrequencyPosteriors, int idxAA, int idxAB, int idxBB) {
-        int numSamples = GLs.size();
-        int numChr = 2*numSamples;
-
-        double[][] logYMatrix = new double[1+numSamples][1+numChr];
-
-        for (int i=0; i <=numSamples; i++)
-            for (int j=0; j <=numChr; j++)
-                logYMatrix[i][j] = Double.NEGATIVE_INFINITY;
-
-        //YMatrix[0][0] = 1.0;
-        logYMatrix[0][0] = 0.0;
-        int j=0;
-
         for ( Map.Entry sample : GLs.entrySet() ) {
-            j++;
 
             if ( !sample.getValue().hasLikelihoods() )
                 continue;
+            Genotype g = GLs.get(sample.getKey());
 
-            //double[] genotypeLikelihoods = MathUtils.normalizeFromLog10(GLs.get(sample).getLikelihoods());
-            double[] genotypeLikelihoods = sample.getValue().getLikelihoods().getAsVector();
-            //double logDenominator = Math.log10(2.0*j*(2.0*j-1));
-            double logDenominator = MathUtils.log10Cache[2*j] + MathUtils.log10Cache[2*j-1];
+            double[] likelihoods = sample.getValue().getLikelihoods().getAsVector();
 
-            // special treatment for k=0: iteration reduces to:
-            //YMatrix[j][0] = YMatrix[j-1][0]*genotypeLikelihoods[GenotypeType.AA.ordinal()];
-            logYMatrix[j][0] = logYMatrix[j-1][0] + genotypeLikelihoods[idxAA];
+            if (MathUtils.sum(likelihoods) <= SUM_GL_THRESH_NOCALL)
+                continue; // regular likelihoods
 
-            for (int k=1; k <= 2*j; k++ ) {
-
-                //double num = (2.0*j-k)*(2.0*j-k-1)*YMatrix[j-1][k] * genotypeLikelihoods[GenotypeType.AA.ordinal()];
-                double logNumerator[];
-                logNumerator = new double[3];
-                if (k < 2*j-1)
-                    logNumerator[0] = MathUtils.log10Cache[2*j-k] + MathUtils.log10Cache[2*j-k-1] + logYMatrix[j-1][k] +
-                            genotypeLikelihoods[idxAA];
-                else
-                    logNumerator[0] = Double.NEGATIVE_INFINITY;
-
-
-                if (k < 2*j)
-                    logNumerator[1] = MathUtils.log10Cache[2*k] + MathUtils.log10Cache[2*j-k]+ logYMatrix[j-1][k-1] +
-                            genotypeLikelihoods[idxAB];
-                else
-                    logNumerator[1] = Double.NEGATIVE_INFINITY;
-
-                if (k > 1)
-                    logNumerator[2] = MathUtils.log10Cache[k] + MathUtils.log10Cache[k-1] + logYMatrix[j-1][k-2] +
-                            genotypeLikelihoods[idxBB];
-                else
-                    logNumerator[2] = Double.NEGATIVE_INFINITY;
-
-                double logNum = MathUtils.softMax(logNumerator);
-
-                //YMatrix[j][k] = num/den;
-                logYMatrix[j][k] = logNum - logDenominator;
-            }
+            ArrayList myAlleles = new ArrayList();
 
+            double qual = Genotype.NO_NEG_LOG_10PERROR;
+            myAlleles.add(Allele.NO_CALL);
+            myAlleles.add(Allele.NO_CALL);
+            //System.out.println(myAlleles.toString());
+            calls.put(sample.getKey(), new Genotype(sample.getKey(), myAlleles, qual, null, g.getAttributes(), false));
         }
-
-        for (int k=0; k <= numChr; k++)
-            log10AlleleFrequencyPosteriors[k] = logYMatrix[j][k] + log10AlleleFrequencyPriors[k];
-
-        return numChr;
+        return calls;
     }
 
     private final static void printLikelihoods(int numChr, double[][] logYMatrix, double[] log10AlleleFrequencyPriors) {
@@ -657,5 +457,4 @@ public class ExactAFCalculationModel extends AlleleFrequencyCalculationModel {
             System.out.printf("  %4d\t%8.2f\t%8.2f\t%8.2f%n", k, logYMatrix[j][k], log10AlleleFrequencyPriors[k], posterior);
         }
     }
-
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
index 41b340058..6d917325e 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/IndelGenotypeLikelihoodsCalculationModel.java
@@ -32,10 +32,11 @@ import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.indels.HaplotypeIndelErrorModel;
 import org.broadinstitute.sting.gatk.walkers.indels.PairHMMIndelErrorModel;
+import org.broadinstitute.sting.utils.BaseUtils;
 import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.StingException;
-import org.broadinstitute.sting.utils.genotype.Haplotype;
 import org.broadinstitute.sting.utils.pileup.ExtendedEventPileupElement;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedExtendedEventPileup;
@@ -70,9 +71,6 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
     // gdebug removeme
     // todo -cleanup
-    private HaplotypeIndelErrorModel model;
-    private boolean useOldWrongHorribleHackedUpLikelihoodModel = false;
-//
     private GenomeLoc lastSiteVisited;
     private ArrayList alleleList;
 
@@ -83,26 +81,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
 
     protected IndelGenotypeLikelihoodsCalculationModel(UnifiedArgumentCollection UAC, Logger logger) {
         super(UAC, logger);
-        if (UAC.GSA_PRODUCTION_ONLY == false) {
-            pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY,
-                    UAC.OUTPUT_DEBUG_INDEL_INFO, UAC.DO_CONTEXT_DEPENDENT_PENALTIES, UAC.dovit, UAC.GET_GAP_PENALTIES_FROM_DATA, UAC.INDEL_RECAL_FILE);
-            useOldWrongHorribleHackedUpLikelihoodModel = false;
-        }
-        else {
-            useOldWrongHorribleHackedUpLikelihoodModel = true;
-            double INSERTION_START_PROBABILITY = 1e-3;
-
-            double INSERTION_END_PROBABILITY = 0.5;
-
-            double ALPHA_DELETION_PROBABILITY = 1e-3;
-
-
-            model = new HaplotypeIndelErrorModel(3, INSERTION_START_PROBABILITY,
-                    INSERTION_END_PROBABILITY,ALPHA_DELETION_PROBABILITY,UAC.INDEL_HAPLOTYPE_SIZE, false, UAC.OUTPUT_DEBUG_INDEL_INFO);
-        }
-
-        pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY,
-                    UAC.OUTPUT_DEBUG_INDEL_INFO, UAC.DO_CONTEXT_DEPENDENT_PENALTIES, UAC.dovit, UAC.GET_GAP_PENALTIES_FROM_DATA, UAC.INDEL_RECAL_FILE);
+        pairModel = new PairHMMIndelErrorModel(UAC.INDEL_GAP_OPEN_PENALTY,UAC.INDEL_GAP_CONTINUATION_PENALTY,UAC.OUTPUT_DEBUG_INDEL_INFO);
         alleleList = new ArrayList();
         getAlleleListFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
         minIndelCountForGenotyping = UAC.MIN_INDEL_COUNT_FOR_GENOTYPING;
@@ -321,7 +300,7 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
             haplotypeMap.clear();
 
             if (getAlleleListFromVCF) {
-                 for( final VariantContext vc_input : tracker.getValues(UAC.alleles) ) {
+                 for( final VariantContext vc_input : tracker.getValues(UAC.alleles, loc) ) {
                       if( vc_input != null &&
                               allowableTypes.contains(vc_input.getType()) &&
                               ref.getLocus().getStart() == vc_input.getStart()) {
@@ -382,20 +361,17 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                 }
             }
         }
-        int eventLength = altAllele.getBaseString().length() - refAllele.getBaseString().length();
-        int hsize = (int)ref.getWindow().size()-Math.abs(eventLength)-1;
-        int numPrefBases= ref.getLocus().getStart()-ref.getWindow().getStart()+1;
 
-        if (useOldWrongHorribleHackedUpLikelihoodModel) {
-            numPrefBases = 20;
-            hsize=80;
-        }
+        final int eventLength = altAllele.getBaseString().length() - refAllele.getBaseString().length();
+        final int hsize = (int)ref.getWindow().size()-Math.abs(eventLength)-1;
+        final int numPrefBases= ref.getLocus().getStart()-ref.getWindow().getStart()+1;
+
         if (DEBUG)
             System.out.format("hsize: %d eventLength: %d refSize: %d, locStart: %d numpr: %d\n",hsize,eventLength,
                     (int)ref.getWindow().size(), loc.getStart(), numPrefBases);
         //System.out.println(eventLength);
-        haplotypeMap = Haplotype.makeHaplotypeListFromAlleles( alleleList, loc.getStart(),
-            ref, hsize, numPrefBases);
+        haplotypeMap = Haplotype.makeHaplotypeListFromAlleles(alleleList, loc.getStart(),
+                ref, hsize, numPrefBases);
 
         // For each sample, get genotype likelihoods based on pileup
         // compute prior likelihoods on haplotypes, and initialize haplotype likelihood matrix with them.
@@ -412,17 +388,9 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
                 pileup = context.getBasePileup();
 
             if (pileup != null ) {
-                double[] genotypeLikelihoods;
-                if (useOldWrongHorribleHackedUpLikelihoodModel)
-                   genotypeLikelihoods = model.computeReadHaplotypeLikelihoods( pileup, haplotypeMap);
-                else
-                    genotypeLikelihoods = pairModel.computeReadHaplotypeLikelihoods( pileup, haplotypeMap, ref, eventLength, getIndelLikelihoodMap());
+                final double[] genotypeLikelihoods = pairModel.computeReadHaplotypeLikelihoods( pileup, haplotypeMap, ref, eventLength, getIndelLikelihoodMap());
 
-
-
-                // which genotype likelihoods correspond to two most likely alleles? By convention, likelihood vector is ordered as for example
-                // for 3 alleles it's 00 01 11 02 12 22
-                 GLs.put(sample.getKey(), new MultiallelicGenotypeLikelihoods(sample.getKey(),
+                GLs.put(sample.getKey(), new MultiallelicGenotypeLikelihoods(sample.getKey(),
                         alleleList,
                         genotypeLikelihoods,
                         getFilteredDepth(pileup)));
@@ -444,4 +412,16 @@ public class IndelGenotypeLikelihoodsCalculationModel extends GenotypeLikelihood
         return indelLikelihoodMap.get();
     }
 
+    // Overload function in GenotypeLikelihoodsCalculationModel so that, for an indel case, we consider a deletion as part of the pileup,
+    // so that per-sample DP will include deletions covering the event.
+    protected int getFilteredDepth(ReadBackedPileup pileup) {
+        int count = 0;
+        for ( PileupElement p : pileup ) {
+            if (p.isDeletion() || BaseUtils.isRegularBase(p.getBase()) )
+                count++;
+        }
+
+        return count;
+    }
+
 }
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
index 477155241..9bdc754e9 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/SNPGenotypeLikelihoodsCalculationModel.java
@@ -26,16 +26,14 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import org.apache.log4j.Logger;
-import org.broadinstitute.sting.commandline.RodBinding;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContextUtils;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.utils.BaseUtils;
+import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.baq.BAQ;
-import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.StingException;
-import org.broadinstitute.sting.utils.genotype.DiploidGenotype;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileupImpl;
@@ -58,25 +56,6 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         useAlleleFromVCF = UAC.GenotypingMode == GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES;
     }
 
-    public static VariantContext getSNPVCFromAllelesRod(RefMetaDataTracker tracker, ReferenceContext ref, boolean requireSNP, Logger logger, final RodBinding allelesBinding) {
-        if ( tracker == null || ref == null || logger == null )
-            throw new ReviewedStingException("Bad arguments: tracker=" + tracker + " ref=" + ref + " logger=" + logger);
-        VariantContext vc = null;
-
-        // search for usable record
-        for( final VariantContext vc_input : tracker.getValues(allelesBinding) ) {
-            if ( vc_input != null && ! vc_input.isFiltered() && (! requireSNP || vc_input.isSNP() )) {
-                if ( vc == null ) {
-                    vc = vc_input;
-                } else {
-                    logger.warn("Multiple valid VCF records detected at site " + ref.getLocus() + ", only considering alleles from first record");
-                }
-            }
-        }
-
-        return vc;
-    }
-
     public Allele getLikelihoods(RefMetaDataTracker tracker,
                                  ReferenceContext ref,
                                  Map contexts,
@@ -96,7 +75,7 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
         if ( alternateAlleleToUse != null ) {
             bestAlternateAllele = alternateAlleleToUse.getBases()[0];
         } else if ( useAlleleFromVCF ) {
-            VariantContext vc = getSNPVCFromAllelesRod(tracker, ref, true, logger, UAC.alleles);
+            VariantContext vc = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, ref, ref.getLocus(), true, logger, UAC.alleles);
 
             // ignore places where we don't have a variant
             if ( vc == null )
@@ -143,8 +122,10 @@ public class SNPGenotypeLikelihoodsCalculationModel extends GenotypeLikelihoodsC
             aList.add(refAllele);
             aList.add(altAllele);
             double[] dlike = new double[]{likelihoods[refGenotype.ordinal()],likelihoods[hetGenotype.ordinal()],likelihoods[homGenotype.ordinal()]} ;
+
+            // normalize in log space so that max element is zero.
             GLs.put(sample.getKey(), new MultiallelicGenotypeLikelihoods(sample.getKey(),
-                    aList,  dlike, getFilteredDepth(pileup)));
+                    aList,  MathUtils.normalizeFromLog10(dlike, false, true), getFilteredDepth(pileup)));
         }
 
         return refAllele;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java
index 500b11360..d88e55687 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UGCallVariants.java
@@ -30,7 +30,6 @@ import org.broadinstitute.sting.commandline.Output;
 import org.broadinstitute.sting.commandline.RodBinding;
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.gatk.datasources.rmd.ReferenceOrderedDataSource;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
 import org.broadinstitute.sting.gatk.walkers.RodWalker;
 import org.broadinstitute.sting.utils.SampleUtils;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
index 7b8045581..f5a107ee7 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedArgumentCollection.java
@@ -143,35 +143,21 @@ public class UnifiedArgumentCollection {
     @Hidden
     @Argument(fullName = "indelHaplotypeSize", shortName = "indelHSize", doc = "Indel haplotype size", required = false)
     public int INDEL_HAPLOTYPE_SIZE = 80;
-    @Hidden
-    @Argument(fullName = "doContextDependentGapPenalties", shortName = "doCDP", doc = "Vary gap penalties by context", required = false)
-     public boolean DO_CONTEXT_DEPENDENT_PENALTIES = true;
+
     //gdebug+
     // experimental arguments, NOT TO BE USED BY ANYONE WHOSE INITIALS AREN'T GDA!!!
-    @Hidden
-    @Argument(fullName = "getGapPenaltiesFromData", shortName = "dataGP", doc = "Vary gap penalties by context - EXPERIMENTAL, DO NO USE", required = false)
-    public boolean GET_GAP_PENALTIES_FROM_DATA = false;
-
-    @Hidden
-    @Argument(fullName="indel_recal_file", shortName="recalFile", required=false, doc="Filename for the input covariates table recalibration .csv file - EXPERIMENTAL, DO NO USE")
-    public File INDEL_RECAL_FILE = new File("indel.recal_data.csv");
+//    @Hidden
+//    @Argument(fullName = "getGapPenaltiesFromData", shortName = "dataGP", doc = "Vary gap penalties by context - EXPERIMENTAL, DO NO USE", required = false)
+//    public boolean GET_GAP_PENALTIES_FROM_DATA = false;
+//
+//    @Hidden
+//    @Argument(fullName="indel_recal_file", shortName="recalFile", required=false, doc="Filename for the input covariates table recalibration .csv file - EXPERIMENTAL, DO NO USE")
+//    public File INDEL_RECAL_FILE = new File("indel.recal_data.csv");
 
     @Hidden
     @Argument(fullName = "indelDebug", shortName = "indelDebug", doc = "Output indel debug info", required = false)
     public boolean OUTPUT_DEBUG_INDEL_INFO = false;
 
-    @Hidden
-    @Argument(fullName = "dovit", shortName = "dovit", doc = "Perform full Viterbi calculation when evaluating the HMM", required = false)
-    public boolean dovit = false;
-
-    @Hidden
-    @Argument(fullName = "GSA_PRODUCTION_ONLY", shortName = "GSA_PRODUCTION_ONLY", doc = "don't ever use me", required = false)
-    public boolean GSA_PRODUCTION_ONLY = false;
-
-    @Hidden
-    @Argument(fullName = "exactCalculation", shortName = "exactCalculation", doc = "expt", required = false)
-    public ExactAFCalculationModel.ExactCalculation EXACT_CALCULATION_TYPE = ExactAFCalculationModel.ExactCalculation.LINEAR_EXPERIMENTAL;
-
     @Hidden
     @Argument(fullName = "ignoreSNPAlleles", shortName = "ignoreSNPAlleles", doc = "expt", required = false)
     public boolean IGNORE_SNP_ALLELES = false;
@@ -191,7 +177,6 @@ public class UnifiedArgumentCollection {
 
         uac.GLmodel = GLmodel;
         uac.AFmodel = AFmodel;
-        uac.EXACT_CALCULATION_TYPE = EXACT_CALCULATION_TYPE;
         uac.heterozygosity = heterozygosity;
         uac.PCR_error = PCR_error;
         uac.GenotypingMode = GenotypingMode;
@@ -209,15 +194,10 @@ public class UnifiedArgumentCollection {
         uac.INDEL_GAP_CONTINUATION_PENALTY = INDEL_GAP_CONTINUATION_PENALTY;
         uac.OUTPUT_DEBUG_INDEL_INFO = OUTPUT_DEBUG_INDEL_INFO;
         uac.INDEL_HAPLOTYPE_SIZE = INDEL_HAPLOTYPE_SIZE;
-        uac.DO_CONTEXT_DEPENDENT_PENALTIES = DO_CONTEXT_DEPENDENT_PENALTIES;
         uac.alleles = alleles;
 
-        uac.GET_GAP_PENALTIES_FROM_DATA = GET_GAP_PENALTIES_FROM_DATA;
-        uac.INDEL_RECAL_FILE = INDEL_RECAL_FILE;
         // todo- arguments to remove
         uac.COVERAGE_AT_WHICH_TO_ABORT = COVERAGE_AT_WHICH_TO_ABORT;
-        uac.dovit = dovit;
-        uac.GSA_PRODUCTION_ONLY = GSA_PRODUCTION_ONLY;
         uac.IGNORE_SNP_ALLELES = IGNORE_SNP_ALLELES;
         
         return uac;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
index d5dbdedd6..428f97e2a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyper.java
@@ -38,7 +38,6 @@ import org.broadinstitute.sting.gatk.walkers.annotator.VariantAnnotatorEngine;
 import org.broadinstitute.sting.gatk.walkers.annotator.interfaces.AnnotatorCompatibleWalker;
 import org.broadinstitute.sting.utils.SampleUtils;
 import org.broadinstitute.sting.utils.baq.BAQ;
-import org.broadinstitute.sting.utils.codecs.snpEff.SnpEffFeature;
 import org.broadinstitute.sting.utils.codecs.vcf.*;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
@@ -127,7 +126,8 @@ public class UnifiedGenotyper extends LocusWalker getDbsnpRodBinding() { return dbsnp.dbsnp; }
-    public RodBinding getSnpEffRodBinding() { return null; }
+    public RodBinding getVariantRodBinding() { return null; }
+    public RodBinding getSnpEffRodBinding() { return null; }
     public List> getCompRodBindings() { return Collections.emptyList(); }
     public List> getResourceRodBindings() { return Collections.emptyList(); }
 
@@ -210,7 +210,7 @@ public class UnifiedGenotyper extends LocusWalker stratifiedContexts, AlignmentContext rawContext) {
         VariantContext vc;
         if ( UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES ) {
-            VariantContext vcInput = SNPGenotypeLikelihoodsCalculationModel.getSNPVCFromAllelesRod(tracker, ref, false, logger, UAC.alleles);
+            VariantContext vcInput = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, ref, rawContext.getLocation(), false, logger, UAC.alleles);
             if ( vcInput == null )
                 return null;
-            vc = new VariantContext("UG_call", vcInput.getChr(), vcInput.getStart(), vcInput.getEnd(), vcInput.getAlleles());
+            vc = new VariantContext("UG_call", vcInput.getChr(), vcInput.getStart(), vcInput.getEnd(), vcInput.getAlleles(), InferredGeneticContext.NO_NEG_LOG_10PERROR, null, null, ref.getBase());
+
         } else {
             // deal with bad/non-standard reference bases
             if ( !Allele.acceptableAlleleBases(new byte[]{ref.getBase()}) )
@@ -544,6 +544,21 @@ public class UnifiedGenotyperEngine {
             AFs[i] = AlleleFrequencyCalculationModel.VALUE_NOT_CALCULATED;
     }
 
+    private final static double[] binomialProbabilityDepthCache = new double[10000];
+    static {
+        for ( int i = 1; i < binomialProbabilityDepthCache.length; i++ ) {
+            binomialProbabilityDepthCache[i] = MathUtils.binomialProbability(0, i, 0.5);
+        }
+    }
+
+    private final double getRefBinomialProb(final int depth) {
+        if ( depth < binomialProbabilityDepthCache.length )
+            return binomialProbabilityDepthCache[depth];
+        else
+            return MathUtils.binomialProbability(0, depth, 0.5);
+    }
+
+
     private VariantCallContext estimateReferenceConfidence(VariantContext vc, Map contexts, double theta, boolean ignoreCoveredSamples, double initialPofRef) {
         if ( contexts == null )
             return null;
@@ -567,7 +582,7 @@ public class UnifiedGenotyperEngine {
                     depth = context.getExtendedEventPileup().size();
             }
 
-            P_of_ref *= 1.0 - (theta / 2.0) * MathUtils.binomialProbability(0, depth, 0.5);
+            P_of_ref *= 1.0 - (theta / 2.0) * getRefBinomialProb(depth);
         }
 
         return new VariantCallContext(vc, QualityUtils.phredScaleErrorRate(1.0 - P_of_ref) >= UAC.STANDARD_CONFIDENCE_FOR_CALLING, false);
@@ -635,7 +650,7 @@ public class UnifiedGenotyperEngine {
             // no extended event pileup
             // if we're genotyping given alleles and we have a requested SNP at this position, do SNP
             if (UAC.GenotypingMode == GenotypeLikelihoodsCalculationModel.GENOTYPING_MODE.GENOTYPE_GIVEN_ALLELES) {
-                VariantContext vcInput = SNPGenotypeLikelihoodsCalculationModel.getSNPVCFromAllelesRod(tracker, refContext, false, logger, UAC.alleles);
+                VariantContext vcInput = UnifiedGenotyperEngine.getVCFromAllelesRod(tracker, refContext, rawContext.getLocation(), false, logger, UAC.alleles);
                 if (vcInput == null)
                     return null;
 
@@ -741,4 +756,23 @@ public class UnifiedGenotyperEngine {
 
         return afcm;
     }
+
+    public static VariantContext getVCFromAllelesRod(RefMetaDataTracker tracker, ReferenceContext ref, GenomeLoc loc, boolean requireSNP, Logger logger, final RodBinding allelesBinding) {
+        if ( tracker == null || ref == null || logger == null )
+            throw new ReviewedStingException("Bad arguments: tracker=" + tracker + " ref=" + ref + " logger=" + logger);
+        VariantContext vc = null;
+
+        // search for usable record
+        for( final VariantContext vc_input : tracker.getValues(allelesBinding, loc) ) {
+            if ( vc_input != null && ! vc_input.isFiltered() && (! requireSNP || vc_input.isSNP() )) {
+                if ( vc == null ) {
+                    vc = vc_input;
+                } else {
+                    logger.warn("Multiple valid VCF records detected in the alleles input file at site " + ref.getLocus() + ", only considering the first record");
+                }
+            }
+        }
+
+        return vc;
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java
index e68aa31e0..3b3f54b05 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/HaplotypeIndelErrorModel.java
@@ -26,9 +26,9 @@
 package org.broadinstitute.sting.gatk.walkers.indels;
 
 import net.sf.samtools.SAMRecord;
+import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
 import org.broadinstitute.sting.utils.QualityUtils;
-import org.broadinstitute.sting.utils.genotype.Haplotype;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
@@ -73,7 +73,7 @@ public class HaplotypeIndelErrorModel {
         baseMatchArray = new double[MAX_CACHED_QUAL+1];
         baseMismatchArray = new double[MAX_CACHED_QUAL+1];
         for (int k=1; k <= MAX_CACHED_QUAL; k++) {
-            double baseProb = QualityUtils.qualToProb(k);
+            double baseProb = QualityUtils.qualToProb((byte)k);
 
 
             baseMatchArray[k] =  probToQual(baseProb);
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
index 2d7969230..68cbd4fb7 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/PairHMMIndelErrorModel.java
@@ -28,9 +28,10 @@ package org.broadinstitute.sting.gatk.walkers.indels;
 import net.sf.samtools.Cigar;
 import net.sf.samtools.CigarElement;
 import net.sf.samtools.CigarOperator;
+import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.utils.Haplotype;
 import org.broadinstitute.sting.utils.MathUtils;
-import org.broadinstitute.sting.utils.genotype.Haplotype;
 import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.pileup.ReadBackedPileup;
 import org.broadinstitute.sting.utils.sam.GATKSAMRecord;
@@ -50,36 +51,8 @@ import org.broadinstitute.sting.oneoffprojects.walkers.IndelCountCovariates.Reca
 
 
 public class PairHMMIndelErrorModel {
-
-
     public static final int BASE_QUAL_THRESHOLD = 20;
 
-
-    private static final int MATCH_OFFSET = 0;
-    private static final int X_OFFSET = 1;
-    private static final int Y_OFFSET = 2;
-
-    private static final int DIAG = 0;
-    private static final int UP = 1;
-    private static final int LEFT = 2;
-
-    private static final int DIAG_GOTO_M = 0;
-    private static final int DIAG_GOTO_X = 1;
-    private static final int DIAG_GOTO_Y = 2;
-
-    private static final int UP_GOTO_M = 4;
-    private static final int UP_GOTO_X = 5;
-    private static final int UP_GOTO_Y = 6;
-
-    private static final int LEFT_GOTO_M = 8;
-    private static final int LEFT_GOTO_X = 9;
-    private static final int LEFT_GOTO_Y = 10;
-
-    private static final int[] ACTIONS_M = {DIAG_GOTO_M, DIAG_GOTO_X, DIAG_GOTO_Y};
-    private static final int[] ACTIONS_X = {UP_GOTO_M, UP_GOTO_X, UP_GOTO_Y};
-    private static final int[] ACTIONS_Y = {LEFT_GOTO_M, LEFT_GOTO_X, LEFT_GOTO_Y};
-
-
     private final double logGapOpenProbability;
     private final double logGapContinuationProbability;
 
@@ -100,36 +73,13 @@ public class PairHMMIndelErrorModel {
     private static final double MIN_GAP_CONT_PENALTY = 10.0;
     private static final double GAP_PENALTY_HRUN_STEP = 1.0; // each increase in hrun decreases gap penalty by this.
 
-
-    private boolean doViterbi = false;
-
-    private final boolean useAffineGapModel = true;
-    private boolean doContextDependentPenalties = false;
-
     private final double[] GAP_OPEN_PROB_TABLE;
     private final double[] GAP_CONT_PROB_TABLE;
 
-    private boolean getGapPenaltiesFromFile = false;
-
-    private int SMOOTHING = 1;
-    private int MAX_QUALITY_SCORE = 50;
-    private int PRESERVE_QSCORES_LESS_THAN = 5;
-
     /////////////////////////////
     // Private Member Variables
     /////////////////////////////
-//copy+
-/*    private RecalDataManager dataManager; // Holds the data HashMap, mostly used by TableRecalibrationWalker to create collapsed data hashmaps
-    private final ArrayList requestedCovariates = new ArrayList(); // List of covariates to be used in this calculation
-    private static final Pattern COMMENT_PATTERN = Pattern.compile("^#.*");
-    private static final Pattern OLD_RECALIBRATOR_HEADER = Pattern.compile("^rg,.*");
-    private static final Pattern COVARIATE_PATTERN = Pattern.compile("^ReadGroup,QualityScore,.*");
-    protected static final String EOF_MARKER = "EOF";
-    private long numReadsWithMalformedColorSpace = 0;
-    private RecalibrationArgumentCollection RAC = new RecalibrationArgumentCollection();
-    private NestedHashMap qualityScoreByFullCovariateKey = new NestedHashMap(); // Caches the result of performSequentialQualityCalculation(..) for all sets of covariate values.
-  */
-//copy-
+
     static {
         LOG_ONE_HALF= -Math.log10(2.0);
         END_GAP_COST = LOG_ONE_HALF;
@@ -145,141 +95,9 @@ public class PairHMMIndelErrorModel {
         }
     }
 
-    public  PairHMMIndelErrorModel(double indelGOP, double indelGCP, boolean deb, boolean doCDP, boolean dovit,boolean gpf, File RECAL_FILE) {
-
-        this(indelGOP, indelGCP, deb, doCDP, dovit);
-        this.getGapPenaltiesFromFile = gpf;
-
-        // read data from recal file
-        // gdebug - start copy from TableRecalibrationWalker
-/*        if (gpf) {
-            boolean sawEOF = false;
-            boolean REQUIRE_EOF = false;
-
-            int lineNumber = 0;
-            boolean foundAllCovariates = false;
-            // Get a list of all available covariates
-            final List> classes = new PluginManager(Covariate.class).getPlugins();
-
-            try {
-                for ( String line : new XReadLines(RECAL_FILE) ) {
-                    lineNumber++;
-                    if ( EOF_MARKER.equals(line) ) {
-                        sawEOF = true;
-                    } else if( COMMENT_PATTERN.matcher(line).matches() || OLD_RECALIBRATOR_HEADER.matcher(line).matches() )  {
-                        ; // Skip over the comment lines, (which start with '#')
-                    }
-                    // Read in the covariates that were used from the input file
-                    else if( COVARIATE_PATTERN.matcher(line).matches() ) { // The line string is either specifying a covariate or is giving csv data
-                        if( foundAllCovariates ) {
-                            throw new UserException.MalformedFile( RECAL_FILE, "Malformed input recalibration file. Found covariate names intermingled with data in file: " + RECAL_FILE );
-                        } else { // Found the covariate list in input file, loop through all of them and instantiate them
-                            String[] vals = line.split(",");
-                            for( int iii = 0; iii < vals.length - 3; iii++ ) { // There are n-3 covariates. The last three items are nObservations, nMismatch, and Qempirical
-                                boolean foundClass = false;
-                                for( Class covClass : classes ) {
-                                    if( (vals[iii] + "Covariate").equalsIgnoreCase( covClass.getSimpleName() ) ) {
-                                        foundClass = true;
-                                        try {
-                                            Covariate covariate = (Covariate)covClass.newInstance();
-                                            requestedCovariates.add( covariate );
-                                        } catch (Exception e) {
-                                            throw new DynamicClassResolutionException(covClass, e);
-                                        }
-
-                                    }
-                                }
-
-                                if( !foundClass ) {
-                                    throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration file. The requested covariate type (" + (vals[iii] + "Covariate") + ") isn't a valid covariate option." );
-                                }
-                            }
-                        }
-
-                    } else { // Found a line of data
-                        if( !foundAllCovariates ) {
-                            foundAllCovariates = true;
-
-                            // At this point all the covariates should have been found and initialized
-                            if( requestedCovariates.size() < 2 ) {
-                                throw new UserException.MalformedFile(RECAL_FILE, "Malformed input recalibration csv file. Covariate names can't be found in file: " + RECAL_FILE );
-                            }
-
-                            final boolean createCollapsedTables = true;
-
-                            // Initialize any covariate member variables using the shared argument collection
-                            for( Covariate cov : requestedCovariates ) {
-                                cov.initialize( RAC );
-                            }
-                            // Initialize the data hashMaps
-                            dataManager = new RecalDataManager( createCollapsedTables, requestedCovariates.size() );
-
-                        }
-                        addCSVData(RECAL_FILE, line); // Parse the line and add the data to the HashMap
-                    }
-                }
-
-            } catch ( FileNotFoundException e ) {
-                throw new UserException.CouldNotReadInputFile(RECAL_FILE, "Can not find input file", e);
-            } catch ( NumberFormatException e ) {
-                throw new UserException.MalformedFile(RECAL_FILE, "Error parsing recalibration data at line " + lineNumber + ". Perhaps your table was generated by an older version of CovariateCounterWalker.");
-            }
-
-            if ( !sawEOF ) {
-                final String errorMessage = "No EOF marker was present in the recal covariates table; this could mean that the file is corrupted or was generated with an old version of the CountCovariates tool.";
-                if ( REQUIRE_EOF )
-                    throw new UserException.MalformedFile(RECAL_FILE, errorMessage);
-            }
-
-            if( dataManager == null ) {
-                throw new UserException.MalformedFile(RECAL_FILE, "Can't initialize the data manager. Perhaps the recal csv file contains no data?");
-            }
-
-            // Create the tables of empirical quality scores that will be used in the sequential calculation
-            dataManager.generateEmpiricalQualities( SMOOTHING, MAX_QUALITY_SCORE );
-        }
-        // debug end copy
-  */
-    }
-    /**
-     * For each covariate read in a value and parse it. Associate those values with the data itself (num observation and num mismatches)
-     */
- /*
-    private void addCSVData(final File file, final String line) {
-        final String[] vals = line.split(",");
-
-        // Check if the data line is malformed, for example if the read group string contains a comma then it won't be parsed correctly
-        if( vals.length != requestedCovariates.size() + 3 ) { // +3 because of nObservations, nMismatch, and Qempirical
-            throw new UserException.MalformedFile(file, "Malformed input recalibration file. Found data line with too many fields: " + line +
-                    " --Perhaps the read group string contains a comma and isn't being parsed correctly.");
-        }
-
-        final Object[] key = new Object[requestedCovariates.size()];
-        Covariate cov;
-        int iii;
-        for( iii = 0; iii < requestedCovariates.size(); iii++ ) {
-            cov = requestedCovariates.get( iii );
-            key[iii] = cov.getValue( vals[iii] );
-        }
-
-        // Create a new datum using the number of observations, number of mismatches, and reported quality score
-        final RecalDatum datum = new RecalDatum( Long.parseLong( vals[iii] ), Long.parseLong( vals[iii + 1] ), Double.parseDouble( vals[1] ), 0.0 );
-        // Add that datum to all the collapsed tables which will be used in the sequential calculation
-        dataManager.addToAllTables( key, datum, PRESERVE_QSCORES_LESS_THAN );
-    }
-
-*/
-    public  PairHMMIndelErrorModel(double indelGOP, double indelGCP, boolean deb, boolean doCDP, boolean dovit) {
-        this(indelGOP, indelGCP, deb, doCDP);
-        this.doViterbi = dovit;
-    }
-
-    public PairHMMIndelErrorModel(double indelGOP, double indelGCP, boolean deb, boolean doCDP) {
-
-
+    public PairHMMIndelErrorModel(double indelGOP, double indelGCP, boolean deb) {
         this.logGapOpenProbability = -indelGOP/10.0; // QUAL to log prob
         this.logGapContinuationProbability = -indelGCP/10.0; // QUAL to log prob
-        this.doContextDependentPenalties = doCDP;
         this.DEBUG = deb;
 
 
@@ -313,132 +131,6 @@ public class PairHMMIndelErrorModel {
 
     }
 
-    private double computeReadLikelihoodGivenHaplotype(byte[] haplotypeBases, byte[] readBases, byte[] readQuals) {
-        final int X_METRIC_LENGTH = readBases.length+1;
-        final int Y_METRIC_LENGTH = haplotypeBases.length+1;
-
-        // initialize path metric and traceback memories for likelihood computation
-        double[][] pathMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        int[][] bestMetricArray = new int[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-
-        pathMetricArray[0][0]= 0;//Double.NEGATIVE_INFINITY;
-
-        for (int i=1; i < X_METRIC_LENGTH; i++) {
-            pathMetricArray[i][0] = 0;
-            bestMetricArray[i][0] = UP;
-        }
-
-        for (int j=1; j < Y_METRIC_LENGTH; j++) {
-            pathMetricArray[0][j] = 0;//logGapOpenProbability + (j-1) * logGapContinuationProbability;
-            bestMetricArray[0][j] = LEFT;
-        }
-
-        for (int indI=1; indI < X_METRIC_LENGTH; indI++) {
-            for (int indJ=1; indJ < Y_METRIC_LENGTH; indJ++) {
-
-                byte x = readBases[indI-1];
-                byte y = haplotypeBases[indJ-1];
-                byte qual = readQuals[indI-1];
-
-                double bestMetric = 0.0;
-                int bestMetricIdx = 0;
-
-                // compute metric for match/mismatch
-                // workaround for reads whose bases quality = 0,
-                if (qual < 1)
-                    qual = 1;
-
-                if (qual > MAX_CACHED_QUAL)
-                    qual = MAX_CACHED_QUAL;
-
-                double pBaseRead =  (x == y)? baseMatchArray[(int)qual]:baseMismatchArray[(int)qual];
-                double[] metrics = new double[3];
-
-                metrics[DIAG] = pathMetricArray[indI-1][indJ-1] + pBaseRead;
-                metrics[UP] = pathMetricArray[indI-1][indJ] + logGapOpenProbability;//(end?0.0:logGapOpenProbability);
-                metrics[LEFT] = pathMetricArray[indI][indJ-1] + logGapOpenProbability;//(end?0.0:logGapOpenProbability);
-
-                if (doViterbi) {
-                    bestMetricIdx = MathUtils.maxElementIndex(metrics);
-                    bestMetric = metrics[bestMetricIdx];
-                }
-                else
-                    bestMetric = MathUtils.softMax(metrics);
-
-                pathMetricArray[indI][indJ] = bestMetric;
-                bestMetricArray[indI][indJ] = bestMetricIdx;
-
-            }
-        }
-
-
-        double bestMetric=0.0;
-        int bestMetricIdx=0,bestI=X_METRIC_LENGTH - 1, bestJ=Y_METRIC_LENGTH - 1;
-
-        for (int i=0; i < X_METRIC_LENGTH; i ++ ) {
-            int j= Y_METRIC_LENGTH-1;
-
-            if (pathMetricArray[i][j] > bestMetric) {
-                bestMetric = pathMetricArray[i][j];
-                bestI = i;
-                bestJ = j;
-            }
-        }
-        for (int j=0; j < Y_METRIC_LENGTH; j++ ) {
-            int i= X_METRIC_LENGTH-1;
-            if (pathMetricArray[i][j] >= bestMetric) {
-                bestMetric = pathMetricArray[i][j];
-                bestI = i;
-                bestJ = j;
-            }
-        }
-
-        if (DEBUG && doViterbi) {
-
-            String haplotypeString = new String (haplotypeBases);
-            String readString = new String(readBases);
-
-
-            int i = bestI;
-            int j = bestJ;
-
-
-            System.out.println("Simple NW");
-
-            while (i >0 || j >0) {
-                bestMetricIdx = bestMetricArray[i][j];
-                System.out.print(bestMetricIdx);
-                if (bestMetricIdx == UP) {
-                    // insert gap in Y
-                    haplotypeString = haplotypeString.substring(0,j)+"-"+haplotypeString.substring(j);
-                    i--;
-                } else if (bestMetricIdx == LEFT) {
-                    readString = readString.substring(0,i)+"-"+readString.substring(i);
-                    j--;
-                }
-                else {
-                    i--; j--;
-                }
-            }
-
-
-
-
-            System.out.println("\nAlignment: ");
-            System.out.println("R:"+readString);
-            System.out.println("H:"+haplotypeString);
-            System.out.println();
-
-
-        }
-        if (DEBUG)
-            System.out.format("Likelihood: %5.4f\n", bestMetric);
-
-        return bestMetric;
-
-
-    }
-
     static private void getContextHomopolymerLength(final byte[] refBytes, int[] hrunArray) {
         // compute forward hrun length, example:
         // AGGTGACCCCCCTGAGAG
@@ -479,14 +171,10 @@ public class PairHMMIndelErrorModel {
         final int Y_METRIC_LENGTH = haplotypeBases.length+1;
 
         // initialize path metric and traceback memories for likelihood computation
-        double[][] matchMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        double[][] XMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        double[][] YMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        int[][] bestActionArrayM = new int[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        int[][] bestActionArrayX = new int[X_METRIC_LENGTH][Y_METRIC_LENGTH];
-        int[][] bestActionArrayY = new int[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+        final double[][] matchMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+        final double[][] XMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
+        final double[][] YMetricArray = new double[X_METRIC_LENGTH][Y_METRIC_LENGTH];
 
-        double c,d;
         matchMetricArray[0][0]= END_GAP_COST;//Double.NEGATIVE_INFINITY;
 
         for (int i=1; i < X_METRIC_LENGTH; i++) {
@@ -494,8 +182,6 @@ public class PairHMMIndelErrorModel {
             matchMetricArray[i][0]  = Double.NEGATIVE_INFINITY;
             YMetricArray[i][0]      = Double.NEGATIVE_INFINITY;
             XMetricArray[i][0]      = END_GAP_COST*(i);//logGapOpenProbability + (i-1)*logGapContinuationProbability;
-
-            bestActionArrayX[i][0] = bestActionArrayY[i][0] = bestActionArrayM[i][0] = UP_GOTO_X;
         }
 
         for (int j=1; j < Y_METRIC_LENGTH; j++) {
@@ -503,188 +189,46 @@ public class PairHMMIndelErrorModel {
             matchMetricArray[0][j]  = Double.NEGATIVE_INFINITY;
             XMetricArray[0][j]      = Double.NEGATIVE_INFINITY;
             YMetricArray[0][j]      = END_GAP_COST*(j);//logGapOpenProbability + (j-1) * logGapContinuationProbability;
-
-            bestActionArrayY[0][j] = bestActionArrayM[0][j] = bestActionArrayX[0][j] = LEFT_GOTO_Y;
         }
 
         for (int indI=1; indI < X_METRIC_LENGTH; indI++) {
-            int im1 = indI-1;
+            final int im1 = indI-1;
             for (int indJ=1; indJ < Y_METRIC_LENGTH; indJ++) {
-                int jm1 = indJ-1;
-                byte x = readBases[im1];
-                byte y = haplotypeBases[jm1];
-                byte qual = readQuals[im1];
-
-                double bestMetric = 0.0;
-                int bestMetricIdx = 0;
-
-                // compute metric for match/mismatch
-                // workaround for reads whose bases quality = 0,
-                if (qual < 1)
-                    qual = 1;
-
-                if (qual > MAX_CACHED_QUAL)
-                    qual = MAX_CACHED_QUAL;
-
-                double pBaseRead =  (x == y)? baseMatchArray[(int)qual]:baseMismatchArray[(int)qual];
-
-
-                double[] metrics = new double[3];
-
-
-                if (doViterbi) {
-                    // update match array
-                    metrics[MATCH_OFFSET] = matchMetricArray[im1][jm1] + pBaseRead;
-                    metrics[X_OFFSET] = XMetricArray[im1][jm1] + pBaseRead;
-                    metrics[Y_OFFSET] = YMetricArray[im1][jm1] + pBaseRead;
-
-                    bestMetricIdx = MathUtils.maxElementIndex(metrics);
-                    bestMetric = metrics[bestMetricIdx];
-                }
-                else
-                    bestMetric = MathUtils.softMax(matchMetricArray[im1][jm1] + pBaseRead, XMetricArray[im1][jm1] + pBaseRead,
-                            YMetricArray[im1][jm1] + pBaseRead);
+                final int jm1 = indJ-1;
+                final byte x = readBases[im1];
+                final byte y = haplotypeBases[jm1];
+                final byte qual = readQuals[im1] < 1 ? 1 : (readQuals[im1] > MAX_CACHED_QUAL ? MAX_CACHED_QUAL : readQuals[im1]);
+                final double pBaseRead =  (x == y)? baseMatchArray[(int)qual]:baseMismatchArray[(int)qual];
 
+                double bestMetric = MathUtils.softMax(matchMetricArray[im1][jm1] + pBaseRead,
+                                                      XMetricArray[im1][jm1] + pBaseRead,
+                                                      YMetricArray[im1][jm1] + pBaseRead);
                 matchMetricArray[indI][indJ] = bestMetric;
-                bestActionArrayM[indI][indJ] = ACTIONS_M[bestMetricIdx];
 
                 // update X array
                 // State X(i,j): X(1:i) aligned to a gap in Y(1:j).
                 // When in last column of X, ie X(1:i) aligned to full Y, we don't want to penalize gaps
 
-                //c = (indJ==Y_METRIC_LENGTH-1? END_GAP_COST: currentGOP[jm1]);
-                //d = (indJ==Y_METRIC_LENGTH-1? END_GAP_COST: currentGCP[jm1]);
-                if (getGapPenaltiesFromFile) {
-                    c = currentGOP[im1];
-                    d = logGapContinuationProbability;
-
-                } else {
-                    c = currentGOP[jm1];
-                    d = currentGCP[jm1];
-                }
-                if (indJ == Y_METRIC_LENGTH-1)
-                    c = d = END_GAP_COST;
-
-                if (doViterbi) {
-                    metrics[MATCH_OFFSET] = matchMetricArray[im1][indJ] + c;
-                    metrics[X_OFFSET] = XMetricArray[im1][indJ] + d;
-                    metrics[Y_OFFSET] = Double.NEGATIVE_INFINITY; //YMetricArray[indI-1][indJ] + logGapOpenProbability;
-
-                    bestMetricIdx = MathUtils.maxElementIndex(metrics);
-                    bestMetric = metrics[bestMetricIdx];
-                }
-                else
-                    bestMetric = MathUtils.softMax(matchMetricArray[im1][indJ] + c, XMetricArray[im1][indJ] + d);
-
+                final double c1 = indJ == Y_METRIC_LENGTH-1 ? END_GAP_COST : currentGOP[jm1];
+                final double d1 = indJ == Y_METRIC_LENGTH-1 ? END_GAP_COST : currentGCP[jm1];
+                bestMetric = MathUtils.softMax(matchMetricArray[im1][indJ] + c1, XMetricArray[im1][indJ] + d1);
                 XMetricArray[indI][indJ] = bestMetric;
-                bestActionArrayX[indI][indJ] = ACTIONS_X[bestMetricIdx];
 
                 // update Y array
                 //c = (indI==X_METRIC_LENGTH-1? END_GAP_COST: currentGOP[jm1]);
                 //d = (indI==X_METRIC_LENGTH-1? END_GAP_COST: currentGCP[jm1]);
-                if (getGapPenaltiesFromFile) {
-                    c = currentGOP[im1];
-                    d = logGapContinuationProbability;
-                }
-                else {
-                    c = currentGOP[jm1];
-                    d = currentGCP[jm1];                        
-                }
-                if (indI == X_METRIC_LENGTH-1)
-                    c = d = END_GAP_COST;
-
-
-
-                if (doViterbi) {
-                    metrics[MATCH_OFFSET] = matchMetricArray[indI][jm1] + c;
-                    metrics[X_OFFSET] = Double.NEGATIVE_INFINITY; //XMetricArray[indI][indJ-1] + logGapOpenProbability;
-                    metrics[Y_OFFSET] = YMetricArray[indI][jm1] + d;
-
-                    bestMetricIdx = MathUtils.maxElementIndex(metrics);
-                    bestMetric = metrics[bestMetricIdx];
-                }
-                else
-                    bestMetric = MathUtils.softMax(matchMetricArray[indI][jm1] + c, YMetricArray[indI][jm1] + d);
-
+                final double c2 = indI == X_METRIC_LENGTH-1 ? END_GAP_COST : currentGOP[jm1];
+                final double d2 = indI == X_METRIC_LENGTH-1 ? END_GAP_COST : currentGCP[jm1];
+                bestMetric = MathUtils.softMax(matchMetricArray[indI][jm1] + c2, YMetricArray[indI][jm1] + d2);
                 YMetricArray[indI][indJ] = bestMetric;
-                bestActionArrayY[indI][indJ] = ACTIONS_Y[bestMetricIdx];
-
-
-
             }
         }
 
-        double bestMetric;
-        double metrics[] = new double[3];
-        int bestTable=0, bestI=X_METRIC_LENGTH - 1, bestJ=Y_METRIC_LENGTH - 1;
-        metrics[MATCH_OFFSET] = matchMetricArray[bestI][bestJ];
-        metrics[X_OFFSET] = XMetricArray[bestI][bestJ];
-        metrics[Y_OFFSET] = YMetricArray[bestI][bestJ];
-        if (doViterbi) {
-            bestTable = MathUtils.maxElementIndex(metrics);
-            bestMetric = metrics[bestTable];
-        }
-        else
-            bestMetric = MathUtils.softMax(metrics);
+        final int bestI = X_METRIC_LENGTH - 1, bestJ = Y_METRIC_LENGTH - 1;
+        final double bestMetric = MathUtils.softMax(matchMetricArray[bestI][bestJ],
+                                                    XMetricArray[bestI][bestJ],
+                                                    YMetricArray[bestI][bestJ]);
 
-        // Do traceback (needed only for debugging!)
-        if (DEBUG && doViterbi) {
-
-            int bestAction;
-            int i = bestI;
-            int j = bestJ;
-
-
-            System.out.println("Affine gap NW");
-
-
-            String haplotypeString = new String (haplotypeBases);
-            String readString = new String(readBases);
-
-
-            while (i >0 || j >0) {
-                if (bestTable == X_OFFSET) {
-                    // insert gap in Y
-                    haplotypeString = haplotypeString.substring(0,j)+"-"+haplotypeString.substring(j);
-                    bestAction = bestActionArrayX[i][j];
-                }
-                else if (bestTable == Y_OFFSET) {
-                    readString = readString.substring(0,i)+"-"+readString.substring(i);
-                    bestAction = bestActionArrayY[i][j];
-
-                }
-                else {
-                    bestAction = bestActionArrayM[i][j];
-                }
-                System.out.print(bestAction);
-
-
-                // bestAction contains action to take at next step
-                // encoding of bestAction: upper 2 bits = direction, lower 2 bits = next table
-
-                // bestTable and nextDirection for next step
-                bestTable = bestAction & 0x3;
-                int nextDirection = bestAction >> 2;
-                if (nextDirection == UP) {
-                    i--;
-                } else if (nextDirection == LEFT) {
-                    j--;
-                } else { //  if (nextDirection == DIAG)
-                    i--; j--;
-                }
-
-            }
-
-
-
-
-            System.out.println("\nAlignment: ");
-            System.out.println("R:"+readString);
-            System.out.println("H:"+haplotypeString);
-            System.out.println();
-
-
-        }
         if (DEBUG)
             System.out.format("Likelihood: %5.4f\n", bestMetric);
 
@@ -707,12 +251,12 @@ public class PairHMMIndelErrorModel {
         }
     }
     public synchronized double[] computeReadHaplotypeLikelihoods(ReadBackedPileup pileup, LinkedHashMap haplotypeMap,
-                                                                   ReferenceContext ref, int eventLength,
-                                                                   HashMap> indelLikelihoodMap){
+                                                                 ReferenceContext ref, int eventLength,
+                                                                 HashMap> indelLikelihoodMap){
 
         int numHaplotypes = haplotypeMap.size();
-        double[][] haplotypeLikehoodMatrix = new double[numHaplotypes][numHaplotypes];
-        double readLikelihoods[][] = new double[pileup.getReads().size()][numHaplotypes];
+        final double readLikelihoods[][] = new double[pileup.size()][numHaplotypes];
+        final int readCounts[] = new int[pileup.size()];
         int readIdx=0;
 
         LinkedHashMap gapOpenProbabilityMap = new LinkedHashMap();
@@ -723,34 +267,35 @@ public class PairHMMIndelErrorModel {
             System.out.println(new String(ref.getBases()));
         }
 
-        if (doContextDependentPenalties && !getGapPenaltiesFromFile)   {
-            // will context dependent probabilities based on homopolymer run. Probabilities are filled based on total complete haplotypes.
-
-
-            for (Allele a: haplotypeMap.keySet()) {
-                Haplotype haplotype = haplotypeMap.get(a);
-                byte[] haplotypeBases = haplotype.getBasesAsBytes();
-                double[] contextLogGapOpenProbabilities = new double[haplotypeBases.length];
-                double[] contextLogGapContinuationProbabilities = new double[haplotypeBases.length];
-
-                // get homopolymer length profile for current haplotype
-                int[] hrunProfile = new int[haplotypeBases.length];
-                getContextHomopolymerLength(haplotypeBases,hrunProfile);
-                if (DEBUG) {
-                    System.out.println("Haplotype bases:");
-                    System.out.println(new String(haplotypeBases));
-                    for (int i=0; i < hrunProfile.length; i++)
-                        System.out.format("%d",hrunProfile[i]);
-                    System.out.println();
-                }
-                fillGapProbabilities(hrunProfile, contextLogGapOpenProbabilities, contextLogGapContinuationProbabilities);
-
-                gapOpenProbabilityMap.put(a,contextLogGapOpenProbabilities);
-                gapContProbabilityMap.put(a,contextLogGapContinuationProbabilities);
+        // will context dependent probabilities based on homopolymer run. Probabilities are filled based on total complete haplotypes.
+        // todo -- refactor into separate function
+        for (Allele a: haplotypeMap.keySet()) {
+            Haplotype haplotype = haplotypeMap.get(a);
+            byte[] haplotypeBases = haplotype.getBasesAsBytes();
+            double[] contextLogGapOpenProbabilities = new double[haplotypeBases.length];
+            double[] contextLogGapContinuationProbabilities = new double[haplotypeBases.length];
 
+            // get homopolymer length profile for current haplotype
+            int[] hrunProfile = new int[haplotypeBases.length];
+            getContextHomopolymerLength(haplotypeBases,hrunProfile);
+            if (DEBUG) {
+                System.out.println("Haplotype bases:");
+                System.out.println(new String(haplotypeBases));
+                for (int i=0; i < hrunProfile.length; i++)
+                    System.out.format("%d",hrunProfile[i]);
+                System.out.println();
             }
+            fillGapProbabilities(hrunProfile, contextLogGapOpenProbabilities, contextLogGapContinuationProbabilities);
+
+            gapOpenProbabilityMap.put(a,contextLogGapOpenProbabilities);
+            gapContProbabilityMap.put(a,contextLogGapContinuationProbabilities);
+
         }
+
         for (PileupElement p: pileup) {
+            // > 1 when the read is a consensus read representing multiple independent observations
+            final boolean isReduced = ReadUtils.isReducedRead(p.getRead());
+            readCounts[readIdx] = isReduced ? p.getReducedCount() : 1;
 
             // check if we've already computed likelihoods for this pileup element (i.e. for this read at this location)
             if (indelLikelihoodMap.containsKey(p)) {
@@ -762,61 +307,20 @@ public class PairHMMIndelErrorModel {
             }
             else {
                 //System.out.format("%d %s\n",p.getRead().getAlignmentStart(), p.getRead().getClass().getName());
-                GATKSAMRecord read = ReadUtils.hardClipAdaptorSequence(p.getRead());
+                SAMRecord read = ReadUtils.hardClipAdaptorSequence(p.getRead());
                 if (read == null)
                     continue;
 
-                if(ReadUtils.is454Read(read) && !getGapPenaltiesFromFile) {
+                if ( isReduced ) {
+                    read = ReadUtils.reducedReadWithReducedQuals(read);
+                }
+
+                if(ReadUtils.is454Read(read)) {
                     continue;
                 }
 
                 double[] recalQuals = null;
 
- /*
-                if (getGapPenaltiesFromFile) {
-                    RecalDataManager.parseSAMRecord( read, RAC );
-
-
-                    recalQuals = new double[read.getReadLength()];
-
-                    //compute all covariate values for this read
-                    final Comparable[][] covariateValues_offset_x_covar =
-                            RecalDataManager.computeCovariates((GATKSAMRecord) read, requestedCovariates);
-                    // For each base in the read
-                    for( int offset = 0; offset < read.getReadLength(); offset++ ) {
-
-                        final Object[] fullCovariateKey = covariateValues_offset_x_covar[offset];
-
-                        Byte qualityScore = (Byte) qualityScoreByFullCovariateKey.get(fullCovariateKey);
-                        if(qualityScore == null)
-                        {
-                            qualityScore = performSequentialQualityCalculation( fullCovariateKey );
-                            qualityScoreByFullCovariateKey.put(qualityScore, fullCovariateKey);
-                        }
-
-                        recalQuals[offset] = -((double)qualityScore)/10.0;
-                    }
-
-                    // for each read/haplotype combination, compute likelihoods, ie -10*log10(Pr(R | Hi))
-                    // = sum_j(-10*log10(Pr(R_j | Hi) since reads are assumed to be independent
-                    if (DEBUG)  {
-                        System.out.format("\n\nStarting read:%s S:%d US:%d E:%d UE:%d C:%s\n",read.getReadName(),
-                                read.getAlignmentStart(),
-                                read.getUnclippedStart(), read.getAlignmentEnd(), read.getUnclippedEnd(),
-                                read.getCigarString());
-
-                        byte[] bases = read.getReadBases();
-                        for (int k = 0; k < recalQuals.length; k++) {
-                            System.out.format("%c",bases[k]);
-                        }
-                        System.out.println();
-
-                        for (int k = 0; k < recalQuals.length; k++) {
-                            System.out.format("%.0f ",recalQuals[k]);
-                        }
-                        System.out.println();
-                    }
-                }        */
                 // get bases of candidate haplotypes that overlap with reads
                 final int trailingBases = 3;
 
@@ -937,11 +441,6 @@ public class PairHMMIndelErrorModel {
                             unclippedReadBases.length-numEndClippedBases);
 
                     double[] recalCDP = null;
-                    if (getGapPenaltiesFromFile) {
-                        recalCDP = Arrays.copyOfRange(recalQuals,numStartClippedBases,
-                                unclippedReadBases.length-numEndClippedBases);
-
-                    }
 
                     if (DEBUG) {
                         System.out.println("Read bases:");
@@ -971,27 +470,9 @@ public class PairHMMIndelErrorModel {
                             System.out.println(new String(haplotypeBases));
                         }
 
-                        Double readLikelihood = 0.0;
-                        if (useAffineGapModel) {
-
-                            double[] currentContextGOP = null;
-                            double[] currentContextGCP = null;
-
-                            if (doContextDependentPenalties) {
-
-                               if (getGapPenaltiesFromFile) {
-                                   readLikelihood = computeReadLikelihoodGivenHaplotypeAffineGaps(haplotypeBases, readBases, readQuals, recalCDP, null);
-
-                               }  else {
-                                   currentContextGOP = Arrays.copyOfRange(gapOpenProbabilityMap.get(a), (int)indStart, (int)indStop);
-                                   currentContextGCP = Arrays.copyOfRange(gapContProbabilityMap.get(a), (int)indStart, (int)indStop);
-                                   readLikelihood = computeReadLikelihoodGivenHaplotypeAffineGaps(haplotypeBases, readBases, readQuals, currentContextGOP, currentContextGCP);
-                               }
-                            }
-
-                        }
-                        else
-                            readLikelihood = computeReadLikelihoodGivenHaplotype(haplotypeBases, readBases, readQuals);
+                        final double[] currentContextGOP = Arrays.copyOfRange(gapOpenProbabilityMap.get(a), (int)indStart, (int)indStop);
+                        final double[] currentContextGCP = Arrays.copyOfRange(gapContProbabilityMap.get(a), (int)indStart, (int)indStop);
+                        final double readLikelihood = computeReadLikelihoodGivenHaplotypeAffineGaps(haplotypeBases, readBases, readQuals, currentContextGOP, currentContextGCP);
 
                         readEl.put(a,readLikelihood);
                         readLikelihoods[readIdx][j++] = readLikelihood;
@@ -1004,7 +485,7 @@ public class PairHMMIndelErrorModel {
 
         if (DEBUG) {
             System.out.println("\nLikelihood summary");
-            for (readIdx=0; readIdx < pileup.getReads().size(); readIdx++) {
+            for (readIdx=0; readIdx < pileup.size(); readIdx++) {
                 System.out.format("Read Index: %d ",readIdx);
                 for (int i=0; i < readLikelihoods[readIdx].length; i++)
                     System.out.format("L%d: %f ",i,readLikelihoods[readIdx][i]);
@@ -1012,123 +493,41 @@ public class PairHMMIndelErrorModel {
             }
 
         }
+
+        return getHaplotypeLikelihoods(numHaplotypes, readCounts, readLikelihoods);
+    }
+
+    private final static double[] getHaplotypeLikelihoods(final int numHaplotypes, final int readCounts[], final double readLikelihoods[][]) {
+        final double[][] haplotypeLikehoodMatrix = new double[numHaplotypes][numHaplotypes];
+
+        // todo: MAD 09/26/11 -- I'm almost certain this calculation can be simplied to just a single loop without the intermediate NxN matrix
         for (int i=0; i < numHaplotypes; i++) {
             for (int j=i; j < numHaplotypes; j++){
                 // combine likelihoods of haplotypeLikelihoods[i], haplotypeLikelihoods[j]
                 // L(Hi, Hj) = sum_reads ( Pr(R|Hi)/2 + Pr(R|Hj)/2)
                 //readLikelihoods[k][j] has log10(Pr(R_k) | H[j] )
-                 for (readIdx=0; readIdx < pileup.getReads().size(); readIdx++) {
-
+                for (int readIdx = 0; readIdx < readLikelihoods.length; readIdx++) {
                     // Compute log10(10^x1/2 + 10^x2/2) = log10(10^x1+10^x2)-log10(2)
                     // First term is approximated by Jacobian log with table lookup.
                     if (Double.isInfinite(readLikelihoods[readIdx][i]) && Double.isInfinite(readLikelihoods[readIdx][j]))
                         continue;
-                    haplotypeLikehoodMatrix[i][j] += ( MathUtils.softMax(readLikelihoods[readIdx][i],
-                            readLikelihoods[readIdx][j]) + LOG_ONE_HALF);
-
+                    final double li = readLikelihoods[readIdx][i];
+                    final double lj = readLikelihoods[readIdx][j];
+                    final int readCount = readCounts[readIdx];
+                    haplotypeLikehoodMatrix[i][j] += readCount * (MathUtils.softMax(li, lj) + LOG_ONE_HALF);
                 }
-
-
             }
         }
 
-        return getHaplotypeLikelihoods(haplotypeLikehoodMatrix);
-
-    }
-
-    public static double[] getHaplotypeLikelihoods(double[][] haplotypeLikehoodMatrix) {
-        int hSize = haplotypeLikehoodMatrix.length;
-        double[] genotypeLikelihoods = new double[hSize*(hSize+1)/2];
-
+        final double[] genotypeLikelihoods = new double[numHaplotypes*(numHaplotypes+1)/2];
         int k=0;
-        double maxElement = Double.NEGATIVE_INFINITY;
-        for (int j=0; j < hSize; j++) {
+        for (int j=0; j < numHaplotypes; j++) {
             for (int i=0; i <= j; i++){
                 genotypeLikelihoods[k++] = haplotypeLikehoodMatrix[i][j];
-                if (haplotypeLikehoodMatrix[i][j] > maxElement)
-                    maxElement = haplotypeLikehoodMatrix[i][j];
             }
         }
 
-        // renormalize
-        for (int i=0; i < genotypeLikelihoods.length; i++)
-            genotypeLikelihoods[i] -= maxElement;
-
-        return genotypeLikelihoods;
+        // renormalize   so that max element is zero.
+        return MathUtils.normalizeFromLog10(genotypeLikelihoods, false, true);
     }
-
-    /**
-     * Implements a serial recalibration of the reads using the combinational table.
-     * First, we perform a positional recalibration, and then a subsequent dinuc correction.
-     *
-     * Given the full recalibration table, we perform the following preprocessing steps:
-     *
-     *   - calculate the global quality score shift across all data [DeltaQ]
-     *   - calculate for each of cycle and dinuc the shift of the quality scores relative to the global shift
-     *      -- i.e., DeltaQ(dinuc) = Sum(pos) Sum(Qual) Qempirical(pos, qual, dinuc) - Qreported(pos, qual, dinuc) / Npos * Nqual
-     *   - The final shift equation is:
-     *
-     *      Qrecal = Qreported + DeltaQ + DeltaQ(pos) + DeltaQ(dinuc) + DeltaQ( ... any other covariate ... )
-     * @param key The list of Comparables that were calculated from the covariates
-     * @return A recalibrated quality score as a byte
-     */
- /*
-    private byte performSequentialQualityCalculation( final Object... key ) {
-
-        final byte qualFromRead = (byte)Integer.parseInt(key[1].toString());
-        final Object[] readGroupCollapsedKey = new Object[1];
-        final Object[] qualityScoreCollapsedKey = new Object[2];
-        final Object[] covariateCollapsedKey = new Object[3];
-
-        // The global quality shift (over the read group only)
-        readGroupCollapsedKey[0] = key[0];
-        final RecalDatum globalRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(0).get( readGroupCollapsedKey ));
-        double globalDeltaQ = 0.0;
-        if( globalRecalDatum != null ) {
-            final double globalDeltaQEmpirical = globalRecalDatum.getEmpiricalQuality();
-            final double aggregrateQReported = globalRecalDatum.getEstimatedQReported();
-            globalDeltaQ = globalDeltaQEmpirical - aggregrateQReported;
-        }
-
-        // The shift in quality between reported and empirical
-        qualityScoreCollapsedKey[0] = key[0];
-        qualityScoreCollapsedKey[1] = key[1];
-        final RecalDatum qReportedRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(1).get( qualityScoreCollapsedKey ));
-        double deltaQReported = 0.0;
-        if( qReportedRecalDatum != null ) {
-            final double deltaQReportedEmpirical = qReportedRecalDatum.getEmpiricalQuality();
-            deltaQReported = deltaQReportedEmpirical - qualFromRead - globalDeltaQ;
-        }
-
-        // The shift in quality due to each covariate by itself in turn
-        double deltaQCovariates = 0.0;
-        double deltaQCovariateEmpirical;
-        covariateCollapsedKey[0] = key[0];
-        covariateCollapsedKey[1] = key[1];
-        for( int iii = 2; iii < key.length; iii++ ) {
-            covariateCollapsedKey[2] =  key[iii]; // The given covariate
-            final RecalDatum covariateRecalDatum = ((RecalDatum)dataManager.getCollapsedTable(iii).get( covariateCollapsedKey ));
-            if( covariateRecalDatum != null ) {
-                deltaQCovariateEmpirical = covariateRecalDatum.getEmpiricalQuality();
-                deltaQCovariates += ( deltaQCovariateEmpirical - qualFromRead - (globalDeltaQ + deltaQReported) );
-            }
-        }
-
-        final double newQuality = qualFromRead + globalDeltaQ + deltaQReported + deltaQCovariates;
-        return QualityUtils.boundQual( (int)Math.round(newQuality), (byte)MAX_QUALITY_SCORE );
-
-        // Verbose printouts used to validate with old recalibrator
-        //if(key.contains(null)) {
-        //    System.out.println( key  + String.format(" => %d + %.2f + %.2f + %.2f + %.2f = %d",
-        //                 qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte));
-        //}
-        //else {
-        //    System.out.println( String.format("%s %s %s %s => %d + %.2f + %.2f + %.2f + %.2f = %d",
-        //                 key.get(0).toString(), key.get(3).toString(), key.get(2).toString(), key.get(1).toString(), qualFromRead, globalDeltaQ, deltaQReported, deltaQPos, deltaQDinuc, newQualityByte) );
-        //}
-
-        //return newQualityByte;
-
-    }
-*/
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java
index e5ad3106d..8bba8eac2 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/indels/SomaticIndelDetectorWalker.java
@@ -68,26 +68,59 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import java.io.*;
 import java.util.*;
 
+
 /**
+ * Tool for calling indels in Tumor-Normal paired sample mode; this tool supports single-sample mode as well,
+ * but this latter functionality is now superceded by UnifiedGenotyper.
+ *
+ * 

  * This is a simple, counts-and-cutoffs based tool for calling indels from aligned (preferrably MSA cleaned) sequencing
- * data. Two output formats supported are: BED format (minimal output, required), and extended output that includes read
- * and mismtach statistics around the calls (tuned on with --verbose). The calls can be performed from a single/pooled sample,
- * or from a matched pair of samples (with --somatic option). In the latter case, two input bam files must be specified,
- * the order is important: indels are called from the second sample ("Tumor") and additionally annotated as germline
- * if even a weak evidence for the same indel, not necessarily a confident call, exists in the first sample ("Normal"), or as somatic
- * if first bam has coverage at the site but no indication for an indel. In the --somatic mode, BED output contains
- * only somatic calls, while --verbose output contains all calls annotated with GERMLINE/SOMATIC keywords.
+ * data. Supported output formats are: BED format, extended verbose output (tab separated), and VCF. The latter two outputs
+ * include additional statistics such as mismtaches and base qualitites around the calls, read strandness (how many
+ * forward/reverse reads support ref and indel alleles) etc. It is highly recommended to use these additional
+ * statistics to perform post-filtering of the calls as the tool is tuned for sensitivity (in other words it will
+ * attempt to "call" anything remotely reasonable based only on read counts and will generate all the additional
+ * metrics for the post-processing tools to make the final decision). The calls are performed by default
+ * from a matched tumor-normal pair of samples. In this case, two (sets of) input bam files must be specified using tagged -I
+ * command line arguments: normal and tumor bam(s) must be passed with -I:normal and -I:tumor arguments,
+ * respectively. Indels are called from the tumor sample and annotated as germline
+ * if even a weak evidence for the same indel, not necessarily a confident call, exists in the normal sample, or as somatic
+ * if normal sample has coverage at the site but no indication for an indel. Note that strictly speaking the calling
+ * is not even attempted in normal sample: if there is an indel in normal that is not detected/does not pass a threshold
+ * in tumor sample, it will not be reported.
  *
- * If any of the general usage of this tool or any of the command-line arguments for this tool are not clear to you,
- * please email asivache at broadinstitute dot org and he will gladly explain everything in more detail.
+ * To make indel calls and associated metrics for a single sample, this tool can be run with --unpaired flag (input
+ * bam tagging is not required in this case, and tags are completely ignored if still used: all input bams will be merged
+ * on the fly and assumed to represent a single sample - this tool does not check for sample id in the read groups).
  *
+ * 
Input

+ * 

+ * Tumor and normal bam files (or single sample bam file(s) in --unpaired mode).
+ * 

+ *
+ * 
Output

+ * 

+ * Indel calls with associated metrics.
+ * 

+ *
+ * 
Examples

+ * 
+ * java -Xmx2g -jar GenomeAnalysisTK.jar \
+ *   -R ref.fasta \
+ *   -T SomaticIndelDetector \
+ *   -o indels.vcf \
+ *   -verbose indels.txt
+ *   -I:normal normal.bam \
+ *   -I:tumor tumor.bam
+ * 

  *
  */
+
 @ReadFilters({Platform454Filter.class, MappingQualityZeroFilter.class, PlatformUnitFilter.class})
 public class SomaticIndelDetectorWalker extends ReadWalker {
 //    @Output
 //    PrintStream out;
-    @Output(doc="File to which variants should be written",required=true)
+    @Output(doc="File to write variants (indels) in VCF format",required=true)
     protected VCFWriter vcf_writer = null;
 
     @Argument(fullName="outputFile", shortName="O", doc="output file name (BED format). DEPRECATED> Use --bed", required=true)
@@ -102,68 +135,80 @@ public class SomaticIndelDetectorWalker extends ReadWalker {
 
     @Hidden
     @Argument(fullName = "genotype_intervals", shortName = "genotype",
-            doc = "Calls will be made at each position within the specified interval(s), whether there is an indel or it's the ref", required = false)
+        doc = "Calls will be made at each position within the specified interval(s), whether there is an indel or not", required = false)
     public String genotypeIntervalsFile = null;
 
     @Hidden
     @Argument(fullName="genotypeIntervalsAreNotSorted", shortName="giNotSorted", required=false,
-            doc="This tool assumes that the genotyping interval list (--genotype_intervals) is sorted; "+
-                "if the list turns out to be unsorted, it will throw an exception.  "+
-                "Use this argument when your interval list is not sorted to instruct the IndelGenotyper "+
-                "to sort and keep it in memory (increases memory usage!).")
+        doc="This tool assumes that the genotyping interval list (--genotype_intervals) is sorted; "+
+            "if the list turns out to be unsorted, it will throw an exception.  "+
+            "Use this argument when your interval list is not sorted to instruct the IndelGenotyper "+
+            "to sort and keep it in memory (increases memory usage!).")
     protected boolean GENOTYPE_NOT_SORTED = false;
 
     @Hidden
-	@Argument(fullName="unpaired", shortName="unpaired",
-			doc="Perform unpaired calls (no somatic status detection)", required=false)
+    @Argument(fullName="unpaired", shortName="unpaired",
+                    doc="Perform unpaired calls (no somatic status detection)", required=false)
     boolean call_unpaired = false;
-	boolean call_somatic ;
+    boolean call_somatic ;
 
-	@Argument(fullName="verboseOutput", shortName="verbose",
-			doc="Verbose output file in text format", required=false)
-	java.io.File verboseOutput = null;
+    @Argument(fullName="verboseOutput", shortName="verbose",
+                    doc="Verbose output file in text format", required=false)
+    java.io.File verboseOutput = null;
 
     @Argument(fullName="bedOutput", shortName="bed",
-            doc="Lightweight bed output file (only positions and events, no stats/annotations)", required=false)
+        doc="Lightweight bed output file (only positions and events, no stats/annotations)", required=false)
     java.io.File bedOutput = null;
 
-	@Argument(fullName="minCoverage", shortName="minCoverage",
-			doc="indel calls will be made only at sites with coverage of minCoverage or more reads; with --somatic this value is applied to tumor sample", required=false)
-	int minCoverage = 6;
+    @Argument(fullName="minCoverage", shortName="minCoverage",
+                    doc="indel calls will be made only at sites with tumor coverage of minCoverage or more reads; "+
+            "with --unpaired (single sample) option, this value is used for minimum sample coverage", required=false)
+    int minCoverage = 6;
 
-	@Argument(fullName="minNormalCoverage", shortName="minNormalCoverage",
-			doc="used only with --somatic;  normal sample must have at least minNormalCoverage or more reads at the site to call germline/somatic indel, otherwise the indel (in tumor) is ignored", required=false)
-	int minNormalCoverage = 4;
+    @Argument(fullName="minNormalCoverage", shortName="minNormalCoverage",
+                    doc="used only in default (somatic) mode;  normal sample must have at least minNormalCoverage "+
+            "or more reads at the site to call germline/somatic indel, otherwise the indel (in tumor) is ignored", required=false)
+    int minNormalCoverage = 4;
 
-	@Argument(fullName="minFraction", shortName="minFraction",
-			doc="Minimum fraction of reads with CONSENSUS indel at a site, out of all reads covering the site, required for making a call"+
-			" (fraction of non-consensus indels at the site is not considered here, see minConsensusFraction)", required=false)
-	double minFraction = 0.3;
+    @Argument(fullName="minFraction", shortName="minFraction",
+                    doc="Minimum fraction of reads with CONSENSUS indel at a site, out of all reads covering the site, required for making a call"+
+                    " (fraction of non-consensus indels at the site is not considered here, see minConsensusFraction)", required=false)
+    double minFraction = 0.3;
 
-	@Argument(fullName="minConsensusFraction", shortName="minConsensusFraction",
-			doc="Indel call is made only if fraction of CONSENSUS indel observations at a site wrt all indel observations at the site exceeds this threshold", required=false)
-	double minConsensusFraction = 0.7;
+    @Argument(fullName="minConsensusFraction", shortName="minConsensusFraction",
+                    doc="Indel call is made only if fraction of CONSENSUS indel observations at a site wrt "+
+            "all indel observations at the site exceeds this threshold", required=false)
+    double minConsensusFraction = 0.7;
 
-	@Argument(fullName="minIndelCount", shortName="minCnt",
-			doc="Minimum count of reads supporting consensus indel required for making the call. "+
-			" This filter supercedes minFraction, i.e. indels with acceptable minFraction at low coverage "+
-			"(minIndelCount not met) will not pass.", required=false)
-	int minIndelCount = 0;
+    @Argument(fullName="minIndelCount", shortName="minCnt",
+                    doc="Minimum count of reads supporting consensus indel required for making the call. "+
+                    " This filter supercedes minFraction, i.e. indels with acceptable minFraction at low coverage "+
+                    "(minIndelCount not met) will not pass.", required=false)
+    int minIndelCount = 0;
 
-	@Argument(fullName="refseq", shortName="refseq",
-			doc="Name of RefSeq transcript annotation file. If specified, indels will be annotated with GENOMIC/UTR/INTRON/CODING and with the gene name", required=false)
-	String RefseqFileName = null;
+    @Argument(fullName="refseq", shortName="refseq",
+                    doc="Name of RefSeq transcript annotation file. If specified, indels will be annotated with "+
+            "GENOMIC/UTR/INTRON/CODING and with the gene name", required=false)
+    String RefseqFileName = null;
 
-    @Argument(fullName="blacklistedLanes", shortName="BL",
-            doc="Name of lanes (platform units) that should be ignored. Reads coming from these lanes will never be seen "+
-                    "by this application, so they will not contribute indels to consider and will not be counted.", required=false)
-    PlatformUnitFilterHelper dummy;
-     @Argument(fullName="indel_debug", shortName="idebug", doc="Detailed printout for debugging, do not turn this on",required=false) Boolean DEBUG = false;
+//@Argument(fullName="blacklistedLanes", shortName="BL",
+//        doc="Name of lanes (platform units) that should be ignored. Reads coming from these lanes will never be seen "+
+//                "by this application, so they will not contribute indels to consider and will not be counted.", required=false)
+//PlatformUnitFilterHelper dummy;
+
+    @Hidden
+    @Argument(fullName="indel_debug", shortName="idebug", doc="Detailed printout for debugging, do not turn this on",
+            required=false) Boolean DEBUG = false;
     @Argument(fullName="window_size", shortName="ws", doc="Size (bp) of the sliding window used for accumulating the coverage. "+
-            "May need to be increased to accomodate longer reads or longer deletions.",required=false) int WINDOW_SIZE = 200;
+            "May need to be increased to accomodate longer reads or longer deletions. A read can be fit into the "+
+            "window if its length on the reference (i.e. read length + length of deletion gap(s) if any) is smaller "+
+            "than the window size. Reads that do not fit will be ignored, so long deletions can not be called "+
+            "if window is too small",required=false) int WINDOW_SIZE = 200;
     @Argument(fullName="maxNumberOfReads",shortName="mnr",doc="Maximum number of reads to cache in the window; if number of reads exceeds this number,"+
                 " the window will be skipped and no calls will be made from it",required=false) int MAX_READ_NUMBER = 10000;
 
+
+
 	private WindowContext tumor_context;
 	private WindowContext normal_context; 
 	private int currentContigIndex = -1;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/PhasingRead.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/PhasingRead.java
index a56c9e21e..63fb33295 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/PhasingRead.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/PhasingRead.java
@@ -37,7 +37,7 @@ public class PhasingRead extends BaseArray {
     public PhasingRead(int length, int mappingQual) {
         super(length);
 
-        this.mappingProb = new PreciseNonNegativeDouble(QualityUtils.qualToProb(mappingQual));
+        this.mappingProb = new PreciseNonNegativeDouble(QualityUtils.qualToProb((byte)mappingQual));
 
         this.baseProbs = new PreciseNonNegativeDouble[length];
         Arrays.fill(this.baseProbs, null);
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/RefSeqDataParser.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/RefSeqDataParser.java
index 55da1c152..f94140814 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/RefSeqDataParser.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/phasing/RefSeqDataParser.java
@@ -44,12 +44,12 @@ public class RefSeqDataParser {
         String nameKeyToUseMultiplePrefix = nameKeyToUse + "_";
 
         Map entriesToNames = new HashMap();
-        Integer numRecords = vc.getAttributeAsIntegerNoException(NUM_RECORDS_KEY);
-        if (numRecords != null) {
+        int numRecords = vc.getAttributeAsInt(NUM_RECORDS_KEY, -1);
+        if (numRecords != -1) {
             boolean done = false;
 
             if (numRecords == 1) { // Check if perhaps the single record doesn't end with "_1":
-                String name = vc.getAttributeAsStringNoException(nameKeyToUse);
+                String name = vc.getAttributeAsString(nameKeyToUse, null);
                 if (name != null) {
                     entriesToNames.put(nameKeyToUse, name);
                     done = true;
@@ -59,14 +59,14 @@ public class RefSeqDataParser {
             if (!done) {
                 for (int i = 1; i <= numRecords; i++) {
                     String key = nameKeyToUseMultiplePrefix + i;
-                    String name = vc.getAttributeAsStringNoException(key);
+                    String name = vc.getAttributeAsString(key, null);
                     if (name != null)
                         entriesToNames.put(key, name);
                 }
             }
         }
         else { // no entry with the # of records:
-            String name = vc.getAttributeAsStringNoException(nameKeyToUse);
+            String name = vc.getAttributeAsString(nameKeyToUse, null);
             if (name != null) {
                 entriesToNames.put(nameKeyToUse, name);
             }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/DocumentationTest.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/DocumentationTest.java
index 933e24784..5b60a9db5 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/DocumentationTest.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/DocumentationTest.java
@@ -42,6 +42,7 @@ import java.util.*;
  *
  * 
Body test

  */
+@Hidden
 public class DocumentationTest extends RodWalker {
     // the docs for the arguments are in the collection
     @ArgumentCollection protected StandardVariantContextInputArgumentCollection variantCollection = new StandardVariantContextInputArgumentCollection();
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java
index 98c8950e3..1bdb70bdd 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java
@@ -76,6 +76,42 @@ import java.util.Map;
  * 
Output

  * 

  * A recalibration table file in CSV format that is used by the TableRecalibration walker.
+ * It is a comma-separated text file relating the desired covariates to the number of such bases and their rate of mismatch in the genome, and its implied empirical quality score.  
+ *
+ * The first 20 lines of such a file is shown below.  
+ * * The file begins with a series of comment lines describing:
+ * ** The number of counted loci
+ * ** The number of counted bases
+ * ** The number of skipped loci and the fraction skipped, due to presence in dbSNP or bad reference bases
+ * 
+ * * After the comments appears a header line indicating which covariates were used as well as the ordering of elements in the subsequent records.  
+ *
+ * * After the header, data records occur one per line until the end of the file. The first several items on a line are the values of the individual covariates and will change
+ * depending on which covariates were specified at runtime. The last three items are the data- that is, number of observations for this combination of covariates, number of 
+ * reference mismatches, and the raw empirical quality score calculated by phred-scaling the mismatch rate.
+ * 
+ * 
+ * # Counted Sites    19451059
+ * # Counted Bases    56582018
+ * # Skipped Sites    82666
+ * # Fraction Skipped 1 / 235 bp
+ * ReadGroup,QualityScore,Cycle,Dinuc,nObservations,nMismatches,Qempirical
+ * SRR006446,11,65,CA,9,1,10
+ * SRR006446,11,48,TA,10,0,40
+ * SRR006446,11,67,AA,27,0,40
+ * SRR006446,11,61,GA,11,1,10
+ * SRR006446,12,34,CA,47,1,17
+ * SRR006446,12,30,GA,52,1,17
+ * SRR006446,12,36,AA,352,1,25
+ * SRR006446,12,17,TA,182,11,12
+ * SRR006446,11,48,TG,2,0,40
+ * SRR006446,11,67,AG,1,0,40
+ * SRR006446,12,34,CG,9,0,40
+ * SRR006446,12,30,GG,43,0,40
+ * ERR001876,4,31,AG,1,0,40
+ * ERR001876,4,31,AT,2,2,1
+ * ERR001876,4,31,CA,1,0,40
+ * 

  * 

  *
  * 
Examples

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java
index 01e8cd321..48cba6a1a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/validation/ValidationAmplicons.java
@@ -61,7 +61,7 @@ import java.util.List;
  * CACGTTCGGcttgtgcagagcctcaaggtcatccagaggtgatAGTTTAGGGCCCTCTCAAGTCTTTCCNGTGCGCATGG[GT/AC*]CAGCCCTGGGCACCTGTNNNNNNNNNNNNNTGCTCATGGCCTTCTAGATTCCCAGGAAATGTCAGAGCTTTTCAAAGCCC
  *
* are amplicon sequences resulting from running the tool. The flags (preceding the sequence itself) can be: - * + *
  * Valid                     // amplicon is valid
  * SITE_IS_FILTERED=1        // validation site is not marked 'PASS' or '.' in its filter field ("you are trying to validate a filtered variant")
  * VARIANT_TOO_NEAR_PROBE=1  // there is a variant too near to the variant to be validated, potentially shifting the mass-spec peak
@@ -72,10 +72,10 @@ import java.util.List;
  * END_TOO_CLOSE,            // variant is too close to the end of the amplicon region to give sequenom a good chance to find a suitable primer
  * NO_VARIANTS_FOUND,        // no variants found within the amplicon region
  * INDEL_OVERLAPS_VALIDATION_SITE, // an insertion or deletion interferes directly with the site to be validated (i.e. insertion directly preceding or postceding, or a deletion that spans the site itself)
- * 

+ *

* *

Examples

- * 

+ * 
  *    java
  *      -jar GenomeAnalysisTK.jar
  *      -T ValidationAmplicons
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java
index 65e3d3e5a..28f4f2a56 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalWalker.java
@@ -55,7 +55,23 @@ import java.util.*;
  *
  * 
Output

  * 

- * Evaluation tables.
+ * Evaluation tables detailing the results of the eval modules which were applied.
+ * For example:
+ * 
+ * output.eval.gatkreport:
+ * ##:GATKReport.v0.1 CountVariants : Counts different classes of variants in the sample
+ * CountVariants  CompRod   CpG      EvalRod  JexlExpression  Novelty  nProcessedLoci  nCalledLoci  nRefLoci  nVariantLoci  variantRate ...
+ * CountVariants  dbsnp     CpG      eval     none            all      65900028        135770       0         135770        0.00206024  ...
+ * CountVariants  dbsnp     CpG      eval     none            known    65900028        47068        0         47068         0.00071423  ...
+ * CountVariants  dbsnp     CpG      eval     none            novel    65900028        88702        0         88702         0.00134601  ...
+ * CountVariants  dbsnp     all      eval     none            all      65900028        330818       0         330818        0.00502000  ...
+ * CountVariants  dbsnp     all      eval     none            known    65900028        120685       0         120685        0.00183133  ...
+ * CountVariants  dbsnp     all      eval     none            novel    65900028        210133       0         210133        0.00318866  ...
+ * CountVariants  dbsnp     non_CpG  eval     none            all      65900028        195048       0         195048        0.00295976  ...
+ * CountVariants  dbsnp     non_CpG  eval     none            known    65900028        73617        0         73617         0.00111710  ...
+ * CountVariants  dbsnp     non_CpG  eval     none            novel    65900028        121431       0         121431        0.00184265  ...
+ * ...
+ * 

  * 

  *
  * 
Examples

@@ -149,12 +165,12 @@ public class VariantEvalWalker extends RodWalker implements Tr
     @Argument(shortName="mvq", fullName="mendelianViolationQualThreshold", doc="Minimum genotype QUAL score for each trio member required to accept a site as a violation", required=false)
     protected double MENDELIAN_VIOLATION_QUAL_THRESHOLD = 50;
 
-    @Argument(fullName="tranchesFile", shortName="tf", doc="The input tranches file describing where to cut the data", required=false)
-    private String TRANCHE_FILENAME = null;
-
     @Argument(fullName="ancestralAlignments", shortName="aa", doc="Fasta file with ancestral alleles", required=false)
     private File ancestralAlignmentsFile = null;
 
+    @Argument(fullName="requireStrictAlleleMatch", shortName="strict", doc="If provided only comp and eval tracks with exactly matching reference and alternate alleles will be counted as overlapping", required=false)
+    private boolean requireStrictAlleleMatch = false;
+
     // Variables
     private Set jexlExpressions = new TreeSet();
 
@@ -226,16 +242,6 @@ public class VariantEvalWalker extends RodWalker implements Tr
         }
         sampleNamesForStratification.add(ALL_SAMPLE_NAME);
 
-        // Add select expressions for anything in the tranches file
-        if ( TRANCHE_FILENAME != null ) {
-            // we are going to build a few select names automatically from the tranches file
-            for ( Tranche t : Tranche.readTranches(new File(TRANCHE_FILENAME)) ) {
-                logger.info("Adding select for all variant above the pCut of : " + t);
-                SELECT_EXPS.add(String.format(VariantRecalibrator.VQS_LOD_KEY + " >= %.2f", t.minVQSLod));
-                SELECT_NAMES.add(String.format("TS-%.2f", t.ts));
-            }
-        }
-
         // Initialize select expressions
         for (VariantContextUtils.JexlVCMatchExp jexl : VariantContextUtils.initializeMatchExps(SELECT_NAMES, SELECT_EXPS)) {
             SortableJexlVCMatchExp sjexl = new SortableJexlVCMatchExp(jexl.name, jexl.exp);
@@ -245,18 +251,13 @@ public class VariantEvalWalker extends RodWalker implements Tr
         // Initialize the set of stratifications and evaluations to use
         stratificationObjects = variantEvalUtils.initializeStratificationObjects(this, NO_STANDARD_STRATIFICATIONS, STRATIFICATIONS_TO_USE);
         Set> evaluationObjects = variantEvalUtils.initializeEvaluationObjects(NO_STANDARD_MODULES, MODULES_TO_USE);
-        boolean usingJEXL = false;
         for ( VariantStratifier vs : getStratificationObjects() ) {
             if ( vs.getClass().getSimpleName().equals("Filter") )
                 byFilterIsEnabled = true;
             else if ( vs.getClass().getSimpleName().equals("Sample") )
                 perSampleIsEnabled = true;
-            usingJEXL = usingJEXL || vs.getClass().equals(JexlExpression.class);
         }
 
-        if ( TRANCHE_FILENAME != null && ! usingJEXL )
-            throw new UserException.BadArgumentValue("tf", "Requires the JexlExpression ST to enabled");
-
         // Initialize the evaluation contexts
         evaluationContexts = variantEvalUtils.initializeEvaluationContexts(stratificationObjects, evaluationObjects, null, null);
 
@@ -378,16 +379,16 @@ public class VariantEvalWalker extends RodWalker implements Tr
         if ( matchingComps.size() == 0 )
             return null;
 
-        // find the comp which matches the alternate allele from eval
+        // find the comp which matches both the reference allele and alternate allele from eval
         Allele altEval = eval.getAlternateAlleles().size() == 0 ? null : eval.getAlternateAllele(0);
         for ( VariantContext comp : matchingComps ) {
             Allele altComp = comp.getAlternateAlleles().size() == 0 ? null : comp.getAlternateAllele(0);
-            if ( (altEval == null && altComp == null) || (altEval != null && altEval.equals(altComp)) )
+            if ( (altEval == null && altComp == null) || (altEval != null && altEval.equals(altComp) && eval.getReference().equals(comp.getReference())) )
                 return comp;
         }
 
-        // if none match, just return the first one
-        return matchingComps.get(0);
+        // if none match, just return the first one unless we require a strict match
+        return (requireStrictAlleleMatch ? null : matchingComps.get(0));
     }
 
     public Integer treeReduce(Integer lhs, Integer rhs) { return null; }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CompOverlap.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CompOverlap.java
index 2ea64c49c..9facb11b5 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CompOverlap.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CompOverlap.java
@@ -22,9 +22,6 @@ public class CompOverlap extends VariantEvaluator implements StandardEval {
     @DataPoint(description = "number of eval SNP sites")
     long nEvalVariants = 0;
 
-    @DataPoint(description = "number of comp SNP sites")
-    long nCompVariants = 0;
-
     @DataPoint(description = "number of eval sites outside of comp sites")
     long novelSites = 0;
 
@@ -75,10 +72,9 @@ public class CompOverlap extends VariantEvaluator implements StandardEval {
     }
 
     public String update2(VariantContext eval, VariantContext comp, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
-        boolean evalIsGood = eval != null && eval.isVariant();
-        boolean compIsGood = comp != null && comp.isNotFiltered() && (eval == null || comp.getType() == eval.getType());
+        boolean evalIsGood = eval != null && eval.isPolymorphic();
+        boolean compIsGood = comp != null && comp.isNotFiltered();
 
-        if (compIsGood) nCompVariants++;           // count the number of comp events
         if (evalIsGood) nEvalVariants++;           // count the number of eval events
 
         if (compIsGood && evalIsGood) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CountVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CountVariants.java
index 59ef3d992..72058ba7b 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CountVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/CountVariants.java
@@ -100,21 +100,22 @@ public class CountVariants extends VariantEvaluator implements StandardEval {
         // So in order to maintain consistency with the previous implementation (and the intention of the original author), I've
         // added in a proxy check for monomorphic status here.
         // Protect against case when vc only as no-calls too - can happen if we strafity by sample and sample as a single no-call.
-       if ( !vc1.isVariant() || (vc1.hasGenotypes() &&  vc1.getHomRefCount() + vc1.getNoCallCount() == vc1.getNSamples()) ) {
+       if ( vc1.isMonomorphic() ) {
             nRefLoci++;
         } else {
              switch (vc1.getType()) {
                 case NO_VARIATION:
+                    // shouldn't get here
                     break;
                 case SNP:
                     nVariantLoci++;
                     nSNPs++;
-                    if (vc1.getAttributeAsBoolean("ISSINGLETON")) nSingletons++;
+                    if (vc1.getAttributeAsBoolean("ISSINGLETON", false)) nSingletons++;
                     break;
                 case MNP:
                     nVariantLoci++;
                     nMNPs++;
-                    if (vc1.getAttributeAsBoolean("ISSINGLETON")) nSingletons++;
+                    if (vc1.getAttributeAsBoolean("ISSINGLETON", false)) nSingletons++;
                     break;
                 case INDEL:
                     nVariantLoci++;
@@ -136,7 +137,7 @@ public class CountVariants extends VariantEvaluator implements StandardEval {
 
         String refStr = vc1.getReference().getBaseString().toUpperCase();
 
-        String aaStr = vc1.hasAttribute("ANCESTRALALLELE") ? vc1.getAttributeAsString("ANCESTRALALLELE").toUpperCase() : null;
+        String aaStr = vc1.hasAttribute("ANCESTRALALLELE") ? vc1.getAttributeAsString("ANCESTRALALLELE", null).toUpperCase() : null;
 //        if (aaStr.equals(".")) {
 //            aaStr = refStr;
 //        }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java
index a476a2680..e69dbfb28 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/GenotypePhasingEvaluator.java
@@ -219,7 +219,8 @@ public class GenotypePhasingEvaluator extends VariantEvaluator {
     }
 
     public static Double getPQ(Genotype gt) {
-        return gt.getAttributeAsDoubleNoException(ReadBackedPhasingWalker.PQ_KEY);
+        Double d = gt.getAttributeAsDouble(ReadBackedPhasingWalker.PQ_KEY, -1);
+        return d == -1 ? null : d;
     }
 
     public static boolean topMatchesTop(AllelePair b1, AllelePair b2) {
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java
index 35fffd815..ffe7c185f 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelLengthHistogram.java
@@ -90,18 +90,19 @@ public class IndelLengthHistogram extends VariantEvaluator {
     public int getComparisonOrder() { return 1; } // need only the evals
 
     public String update1(VariantContext vc1, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
-        if ( ! vc1.isBiallelic() && vc1.isIndel() ) {
-            //veWalker.getLogger().warn("[IndelLengthHistogram] Non-biallelic indel at "+ref.getLocus()+" ignored.");
-            return vc1.toString(); // biallelic sites are output
-        }
 
-        if ( vc1.isIndel() ) {
+        if ( vc1.isIndel() && vc1.isPolymorphic() ) {
+
+            if ( ! vc1.isBiallelic() ) {
+                //veWalker.getLogger().warn("[IndelLengthHistogram] Non-biallelic indel at "+ref.getLocus()+" ignored.");
+                return vc1.toString(); // biallelic sites are output
+            }
+
+            // only count simple insertions/deletions, not complex indels
             if ( vc1.isSimpleInsertion() ) {
                 indelHistogram.update(vc1.getAlternateAllele(0).length());
             } else if ( vc1.isSimpleDeletion() ) {
                 indelHistogram.update(-vc1.getReference().length());
-            } else {
-                throw new ReviewedStingException("Indel type that is not insertion or deletion.");
             }
         }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelStatistics.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelStatistics.java
index fc347339d..f70e6c2de 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelStatistics.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/IndelStatistics.java
@@ -270,7 +270,7 @@ public class IndelStatistics extends VariantEvaluator {
 
     public String update1(VariantContext eval, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
 
-        if (eval != null ) {
+        if (eval != null && eval.isPolymorphic()) {
             if ( indelStats == null ) {
                 indelStats = new IndelStats(eval);
             }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/SimpleMetricsByAC.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/SimpleMetricsByAC.java
index d466645ea..2d0163206 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/SimpleMetricsByAC.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/SimpleMetricsByAC.java
@@ -120,7 +120,7 @@ public class SimpleMetricsByAC extends VariantEvaluator implements StandardEval
             if ( eval.hasGenotypes() )
                 ac = eval.getChromosomeCount(eval.getAlternateAllele(0));
             else if ( eval.hasAttribute("AC") ) {
-                ac = Integer.valueOf(eval.getAttributeAsString("AC"));
+                ac = eval.getAttributeAsInt("AC", -1);
             }
 
             if ( ac != -1 ) {
@@ -166,7 +166,7 @@ public class SimpleMetricsByAC extends VariantEvaluator implements StandardEval
                 }
             }
 
-            if ( eval.isSNP() && eval.isBiallelic() && metrics != null ) {
+            if ( eval.isSNP() && eval.isBiallelic() && eval.isPolymorphic() && metrics != null ) {
                 metrics.incrValue(eval);
             }
         }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ThetaVariantEvaluator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ThetaVariantEvaluator.java
index ec43cbd55..e51623c3c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ThetaVariantEvaluator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ThetaVariantEvaluator.java
@@ -37,77 +37,74 @@ public class ThetaVariantEvaluator extends VariantEvaluator {
     }
 
     public String update1(VariantContext vc, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
-        if (vc == null || !vc.isSNP() || !vc.hasGenotypes()) {
+        if (vc == null || !vc.isSNP() || !vc.hasGenotypes() || vc.isMonomorphic()) {
             return null; //no interesting sites
         }
 
-        if (vc.hasGenotypes()) {
+        //this maps allele to a count
+        ConcurrentMap alleleCounts = new ConcurrentHashMap();
 
-            //this maps allele to a count
-            ConcurrentMap alleleCounts = new ConcurrentHashMap();
+        int numHetsHere = 0;
+        float numGenosHere = 0;
+        int numIndsHere = 0;
 
-            int numHetsHere = 0;
-            float numGenosHere = 0;
-            int numIndsHere = 0;
+        for (Genotype genotype : vc.getGenotypes().values()) {
+            numIndsHere++;
+            if (!genotype.isNoCall()) {
+                //increment stats for heterozygosity
+                if (genotype.isHet()) {
+                    numHetsHere++;
+                }
 
-            for (Genotype genotype : vc.getGenotypes().values()) {
-                numIndsHere++;
-                if (!genotype.isNoCall()) {
-                    //increment stats for heterozygosity
-                    if (genotype.isHet()) {
-                        numHetsHere++;
-                    }
+                numGenosHere++;
+                //increment stats for pairwise mismatches
 
-                    numGenosHere++;
-                    //increment stats for pairwise mismatches
-
-                    for (Allele allele : genotype.getAlleles()) {
-                        if (allele.isNonNull() && allele.isCalled()) {
-                            String alleleString = allele.toString();
-                            alleleCounts.putIfAbsent(alleleString, 0);
-                            alleleCounts.put(alleleString, alleleCounts.get(alleleString) + 1);
-                        }
+                for (Allele allele : genotype.getAlleles()) {
+                    if (allele.isNonNull() && allele.isCalled()) {
+                        String alleleString = allele.toString();
+                        alleleCounts.putIfAbsent(alleleString, 0);
+                        alleleCounts.put(alleleString, alleleCounts.get(alleleString) + 1);
                     }
                 }
             }
-            if (numGenosHere > 0) {
-                //only if have one called genotype at least
-                this.numSites++;
+        }
+        if (numGenosHere > 0) {
+            //only if have one called genotype at least
+            this.numSites++;
 
-                this.totalHet += numHetsHere / numGenosHere;
+            this.totalHet += numHetsHere / numGenosHere;
 
-                //compute based on num sites
-                float harmonicFactor = 0;
-                for (int i = 1; i <= numIndsHere; i++) {
-                    harmonicFactor += 1.0 / i;
-                }
-                this.thetaRegionNumSites += 1.0 / harmonicFactor;
+            //compute based on num sites
+            float harmonicFactor = 0;
+            for (int i = 1; i <= numIndsHere; i++) {
+                harmonicFactor += 1.0 / i;
+            }
+            this.thetaRegionNumSites += 1.0 / harmonicFactor;
 
-                //now compute pairwise mismatches
-                float numPairwise = 0;
-                float numDiffs = 0;
-                for (String allele1 : alleleCounts.keySet()) {
-                    int allele1Count = alleleCounts.get(allele1);
+            //now compute pairwise mismatches
+            float numPairwise = 0;
+            float numDiffs = 0;
+            for (String allele1 : alleleCounts.keySet()) {
+                int allele1Count = alleleCounts.get(allele1);
 
-                    for (String allele2 : alleleCounts.keySet()) {
-                        if (allele1.compareTo(allele2) < 0) {
-                            continue;
-                        }
-                        if (allele1 .compareTo(allele2) == 0) {
-                            numPairwise += allele1Count * (allele1Count - 1) * .5;
+                for (String allele2 : alleleCounts.keySet()) {
+                    if (allele1.compareTo(allele2) < 0) {
+                        continue;
+                    }
+                    if (allele1 .compareTo(allele2) == 0) {
+                        numPairwise += allele1Count * (allele1Count - 1) * .5;
 
-                        }
-                        else {
-                            int allele2Count = alleleCounts.get(allele2);
-                            numPairwise += allele1Count * allele2Count;
-                            numDiffs += allele1Count * allele2Count;
-                        }
+                    }
+                    else {
+                        int allele2Count = alleleCounts.get(allele2);
+                        numPairwise += allele1Count * allele2Count;
+                        numDiffs += allele1Count * allele2Count;
                     }
                 }
+            }
 
-                if (numPairwise > 0) {
-                    this.totalAvgDiffs += numDiffs / numPairwise;
-                }
+            if (numPairwise > 0) {
+                this.totalAvgDiffs += numDiffs / numPairwise;
             }
         }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/TiTvVariantEvaluator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/TiTvVariantEvaluator.java
index be957abd7..9b6e145e6 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/TiTvVariantEvaluator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/TiTvVariantEvaluator.java
@@ -40,7 +40,7 @@ public class TiTvVariantEvaluator extends VariantEvaluator implements StandardEv
     }
 
     public void updateTiTv(VariantContext vc, boolean updateStandard) {
-        if (vc != null && vc.isSNP() && vc.isBiallelic()) {
+        if (vc != null && vc.isSNP() && vc.isBiallelic() && vc.isPolymorphic()) {
             if (VariantContextUtils.isTransition(vc)) {
                 if (updateStandard) nTiInComp++;
                 else nTi++;
@@ -49,18 +49,14 @@ public class TiTvVariantEvaluator extends VariantEvaluator implements StandardEv
                 else nTv++;
             }
 
-            String refStr = vc.getReference().getBaseString().toUpperCase();
-            String aaStr = vc.getAttributeAsString("ANCESTRALALLELE").toUpperCase();
-
-            if (aaStr != null && !aaStr.equalsIgnoreCase("null") && !aaStr.equals(".")) {
-                BaseUtils.BaseSubstitutionType aaSubType = BaseUtils.SNPSubstitutionType(aaStr.getBytes()[0], vc.getAlternateAllele(0).getBases()[0]);
-
-                //System.out.println(refStr + " " + vc.getAttributeAsString("ANCESTRALALLELE").toUpperCase() + " " + aaSubType);
-
-                if (aaSubType == BaseUtils.BaseSubstitutionType.TRANSITION) {
-                    nTiDerived++;
-                } else if (aaSubType == BaseUtils.BaseSubstitutionType.TRANSVERSION) {
-                    nTvDerived++;
+            if (vc.hasAttribute("ANCESTRALALLELE")) {
+                final String aaStr = vc.getAttributeAsString("ANCESTRALALLELE", "null").toUpperCase();
+                if ( ! aaStr.equals(".") ) {
+                    switch ( BaseUtils.SNPSubstitutionType(aaStr.getBytes()[0], vc.getAlternateAllele(0).getBases()[0] ) ) {
+                        case TRANSITION: nTiDerived++; break;
+                        case TRANSVERSION: nTvDerived++; break;
+                        default: break;
+                    }
                 }
             }
         }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ValidationReport.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ValidationReport.java
index 9c331b577..c60586017 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ValidationReport.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/ValidationReport.java
@@ -117,7 +117,8 @@ public class ValidationReport extends VariantEvaluator implements StandardEval {
     public SiteStatus calcSiteStatus(VariantContext vc) {
         if ( vc == null ) return SiteStatus.NO_CALL;
         if ( vc.isFiltered() ) return SiteStatus.FILTERED;
-        if ( ! vc.isVariant() ) return SiteStatus.MONO;
+        if ( vc.isMonomorphic() ) return SiteStatus.MONO;
+        if ( vc.hasGenotypes() ) return SiteStatus.POLY;  // must be polymorphic if isMonomorphic was false and there are genotypes
 
         if ( vc.hasAttribute(VCFConstants.ALLELE_COUNT_KEY) ) {
             int ac = 0;
@@ -130,10 +131,8 @@ public class ValidationReport extends VariantEvaluator implements StandardEval {
 ////                System.out.printf("  ac = %d%n", ac);
             }
             else
-                ac = vc.getAttributeAsInt(VCFConstants.ALLELE_COUNT_KEY);
+                ac = vc.getAttributeAsInt(VCFConstants.ALLELE_COUNT_KEY, 0);
             return ac > 0 ? SiteStatus.POLY : SiteStatus.MONO;
-        } else if ( vc.hasGenotypes() ) {
-            return vc.isPolymorphic() ? SiteStatus.POLY : SiteStatus.MONO;
         } else {
             return TREAT_ALL_SITES_IN_EVAL_VCF_AS_CALLED ? SiteStatus.POLY : SiteStatus.NO_CALL; // we can't figure out what to do
             //return SiteStatus.NO_CALL; // we can't figure out what to do
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantQualityScore.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantQualityScore.java
index b6ad55b18..263227938 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantQualityScore.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/evaluators/VariantQualityScore.java
@@ -232,7 +232,7 @@ public class VariantQualityScore extends VariantEvaluator {
     public String update1(VariantContext eval, RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
         final String interesting = null;
 
-        if( eval != null && eval.isSNP() && eval.isBiallelic() ) { //BUGBUG: only counting biallelic sites (revisit what to do with triallelic sites)
+        if( eval != null && eval.isSNP() && eval.isBiallelic() && eval.isPolymorphic() ) { //BUGBUG: only counting biallelic sites (revisit what to do with triallelic sites)
             if( titvStats == null ) { titvStats = new TiTvStats(); }
             titvStats.incrValue(eval.getPhredScaledQual(), VariantContextUtils.isTransition(eval));
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleCount.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleCount.java
index 3cc22cc52..c7bea93b2 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleCount.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleCount.java
@@ -44,7 +44,7 @@ public class AlleleCount extends VariantStratifier {
         if (eval != null) {
             int AC = -1;
             if ( eval.hasAttribute("AC") && eval.getAttribute("AC") instanceof Integer ) {
-                AC = eval.getAttributeAsInt("AC");
+                AC = eval.getAttributeAsInt("AC", 0);
             } else if ( eval.isVariant() ) {
                 for (Allele allele : eval.getAlternateAlleles())
                     AC = Math.max(AC, eval.getChromosomeCount(allele));
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleFrequency.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleFrequency.java
index 3d2dda651..cd2b8e475 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleFrequency.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/AlleleFrequency.java
@@ -28,7 +28,7 @@ public class AlleleFrequency extends VariantStratifier {
 
         if (eval != null) {
             try {
-                relevantStates.add(String.format("%.3f", (5.0 * MathUtils.round(eval.getAttributeAsDouble("AF") / 5.0, 3))));
+                relevantStates.add(String.format("%.3f", (5.0 * MathUtils.round(eval.getAttributeAsDouble("AF", 0.0) / 5.0, 3))));
             } catch (Exception e) {
                 return relevantStates;
             }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/Degeneracy.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/Degeneracy.java
index 3223626c0..91c96e490 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/Degeneracy.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/Degeneracy.java
@@ -90,8 +90,8 @@ public class Degeneracy extends VariantStratifier {
             Integer frame = null;
 
             if (eval.hasAttribute("refseq.functionalClass")) {
-                aa = eval.getAttributeAsString("refseq.variantAA");
-                frame = eval.getAttributeAsInt("refseq.frame");
+                aa = eval.getAttributeAsString("refseq.variantAA", null);
+                frame = eval.getAttributeAsInt("refseq.frame", 0);
             } else if (eval.hasAttribute("refseq.functionalClass_1")) {
                 int annotationId = 1;
                 String key;
@@ -99,7 +99,7 @@ public class Degeneracy extends VariantStratifier {
                 do {
                     key = String.format("refseq.functionalClass_%d", annotationId);
 
-                    String newtype = eval.getAttributeAsString(key);
+                    String newtype = eval.getAttributeAsString(key, null);
 
                     if ( newtype != null &&
                             ( type == null ||
@@ -109,13 +109,13 @@ public class Degeneracy extends VariantStratifier {
                         type = newtype;
 
                         String aakey = String.format("refseq.variantAA_%d", annotationId);
-                        aa = eval.getAttributeAsString(aakey);
+                        aa = eval.getAttributeAsString(aakey, null);
 
                         if (aa != null) {
                             String framekey = String.format("refseq.frame_%d", annotationId);
 
                             if (eval.hasAttribute(framekey)) {
-                                frame = eval.getAttributeAsInt(framekey);
+                                frame = eval.getAttributeAsInt(framekey, 0);
                             }
                         }
                     }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/FunctionalClass.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/FunctionalClass.java
index 193a65591..f5dcf527a 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/FunctionalClass.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/FunctionalClass.java
@@ -2,6 +2,7 @@ package org.broadinstitute.sting.gatk.walkers.varianteval.stratifications;
 
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
 import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.annotator.SnpEff;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.util.ArrayList;
@@ -11,25 +12,34 @@ import java.util.List;
  * Stratifies by nonsense, missense, silent, and all annotations in the input ROD, from the INFO field annotation.
  */
 public class FunctionalClass extends VariantStratifier {
-    @Override
-    public void initialize() {
-        states.add("all");
-        states.add("silent");
-        states.add("missense");
-        states.add("nonsense");
+
+    public enum FunctionalType {
+        silent,
+        missense,
+        nonsense
     }
 
 
-    public List getRelevantStates(ReferenceContext ref, RefMetaDataTracker tracker, VariantContext comp, String compName, VariantContext eval, String evalName, String sampleName) {
+    @Override
+    public void initialize() {
+        states.add("all");
+        for ( FunctionalType type : FunctionalType.values() )
+            states.add(type.name());
+    }
+
+
+public List getRelevantStates(ReferenceContext ref, RefMetaDataTracker tracker, VariantContext comp, String compName, VariantContext eval, String evalName, String sampleName) {
         ArrayList relevantStates = new ArrayList();
 
         relevantStates.add("all");
 
         if (eval != null && eval.isVariant()) {
-            String type = null;
+            FunctionalType type = null;
 
             if (eval.hasAttribute("refseq.functionalClass")) {
-                type = eval.getAttributeAsString("refseq.functionalClass");
+                try {
+                    type = FunctionalType.valueOf(eval.getAttributeAsString("refseq.functionalClass", null));
+                } catch ( Exception e ) {} // don't error out if the type isn't supported
             } else if (eval.hasAttribute("refseq.functionalClass_1")) {
                 int annotationId = 1;
                 String key;
@@ -37,24 +47,36 @@ public class FunctionalClass extends VariantStratifier {
                 do {
                     key = String.format("refseq.functionalClass_%d", annotationId);
 
-                    String newtype = eval.getAttributeAsString(key);
-
-                    if ( newtype != null && !newtype.equalsIgnoreCase("null") &&
-                         ( type == null ||
-                         ( type.equals("silent") && !newtype.equals("silent") ) ||
-                         ( type.equals("missense") && newtype.equals("nonsense") ) )
-                       ) {
-                        type = newtype;
+                    String newtypeStr = eval.getAttributeAsString(key, null);
+                    if ( newtypeStr != null && !newtypeStr.equalsIgnoreCase("null") ) {
+                        try {
+                            FunctionalType newType = FunctionalType.valueOf(newtypeStr);
+                            if ( type == null ||
+                                ( type == FunctionalType.silent && newType != FunctionalType.silent ) ||
+                                ( type == FunctionalType.missense && newType == FunctionalType.nonsense ) ) {
+                                type = newType;
+                            }
+                        } catch ( Exception e ) {} // don't error out if the type isn't supported
                     }
 
                     annotationId++;
                 } while (eval.hasAttribute(key));
+
+            } else if ( eval.hasAttribute(SnpEff.InfoFieldKey.FUNCTIONAL_CLASS_KEY.getKeyName()) ) {
+                try {
+                    SnpEff.EffectFunctionalClass snpEffFunctionalClass = SnpEff.EffectFunctionalClass.valueOf(eval.getAttribute(SnpEff.InfoFieldKey.FUNCTIONAL_CLASS_KEY.getKeyName()).toString());
+                    if ( snpEffFunctionalClass == SnpEff.EffectFunctionalClass.NONSENSE )
+                        type = FunctionalType.nonsense;
+                    else if ( snpEffFunctionalClass == SnpEff.EffectFunctionalClass.MISSENSE )
+                        type = FunctionalType.missense;
+                    else if ( snpEffFunctionalClass == SnpEff.EffectFunctionalClass.SILENT )
+                        type = FunctionalType.silent;
+                    }
+                catch ( Exception e ) {} // don't error out if the type isn't supported
             }
 
-            if (type != null) {
-                if      (type.equals("silent"))   { relevantStates.add("silent");   }
-                else if (type.equals("missense")) { relevantStates.add("missense"); }
-                else if (type.equals("nonsense")) { relevantStates.add("nonsense"); }
+            if ( type != null ) {
+                relevantStates.add(type.name());
             }
         }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java
index 3cc039141..6a057a456 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/util/VariantEvalUtils.java
@@ -277,7 +277,7 @@ public class VariantEvalUtils {
      * @return a new VariantContext with just the requested samples
      */
     public VariantContext getSubsetOfVariantContext(VariantContext vc, Collection sampleNames) {
-        VariantContext vcsub = vc.subContextFromGenotypes(vc.getGenotypes(sampleNames).values());
+        VariantContext vcsub = vc.subContextFromGenotypes(vc.getGenotypes(sampleNames).values(), vc.getAlleles());
 
         HashMap newAts = new HashMap(vcsub.getAttributes());
 
@@ -354,7 +354,7 @@ public class VariantEvalUtils {
 
     private void addMapping(HashMap> mappings, String sample, VariantContext vc) {
         if ( !mappings.containsKey(sample) )
-            mappings.put(sample, new HashSet());
+            mappings.put(sample, new LinkedHashSet());
         mappings.get(sample).add(vc);
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/TrainingSet.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/TrainingSet.java
new file mode 100755
index 000000000..5f688d001
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/TrainingSet.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2011 The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
+ * THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.walkers.variantrecalibration;
+
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.commandline.RodBinding;
+import org.broadinstitute.sting.commandline.Tags;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+/**
+ * Created by IntelliJ IDEA.
+ * User: rpoplin
+ * Date: 3/12/11
+ */
+
+public class TrainingSet {
+
+    public RodBinding rodBinding;
+    public boolean isKnown = false;
+    public boolean isTraining = false;
+    public boolean isAntiTraining = false;
+    public boolean isTruth = false;
+    public boolean isConsensus = false;
+    public double prior = 0.0;
+
+    protected final static Logger logger = Logger.getLogger(TrainingSet.class);
+
+    public TrainingSet( final RodBinding rodBinding) {
+        this.rodBinding = rodBinding;
+
+        final Tags tags = rodBinding.getTags();
+        final String name = rodBinding.getName();
+
+        // Parse the tags to decide which tracks have which properties
+        if( tags != null ) {
+            isKnown = tags.containsKey("known") && tags.getValue("known").equals("true");
+            isTraining = tags.containsKey("training") && tags.getValue("training").equals("true");
+            isAntiTraining = tags.containsKey("bad") && tags.getValue("bad").equals("true");
+            isTruth = tags.containsKey("truth") && tags.getValue("truth").equals("true");
+            isConsensus = tags.containsKey("consensus") && tags.getValue("consensus").equals("true");
+            prior = ( tags.containsKey("prior") ? Double.parseDouble(tags.getValue("prior")) : prior );
+        }
+
+        // Report back to the user which tracks were found and the properties that were detected
+        if( !isConsensus && !isAntiTraining ) {
+            logger.info( String.format( "Found %s track: \tKnown = %s \tTraining = %s \tTruth = %s \tPrior = Q%.1f", name, isKnown, isTraining, isTruth, prior) );
+        } else if( isConsensus ) {
+            logger.info( String.format( "Found consensus track: %s", name) );
+        } else {
+            logger.info( String.format( "Found bad sites training track: %s", name) );
+        }
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VQSRCalibrationCurve.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VQSRCalibrationCurve.java
index bc7252ec2..04ba3ff14 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VQSRCalibrationCurve.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VQSRCalibrationCurve.java
@@ -115,7 +115,7 @@ public class VQSRCalibrationCurve {
         if ( vc.isFiltered() )
             return 0.0;
         else if ( vc.hasAttribute(VQSRQualKey) ) {
-            double qual = vc.getAttributeAsDouble(VQSRQualKey);
+            double qual = vc.getAttributeAsDouble(VQSRQualKey, 0.0);
             return probTrueVariant(qual);
         } else {
             throw new UserException.VariantContextMissingRequiredField(VQSRQualKey, vc);
@@ -143,7 +143,7 @@ public class VQSRCalibrationCurve {
             for ( int i = 0; i < log10Likelihoods.length; i++) {
                 double p = Math.pow(10, log10Likelihoods[i]);
                 double q = alpha * p + (1-alpha) * noInfoPr;
-                if ( DEBUG ) System.out.printf("  vqslod = %.2f, p = %.2e, alpha = %.2e, q = %.2e%n", vc.getAttributeAsDouble(VQSRQualKey), p, alpha, q);
+                if ( DEBUG ) System.out.printf("  vqslod = %.2f, p = %.2e, alpha = %.2e, q = %.2e%n", vc.getAttributeAsDouble(VQSRQualKey, 0.0), p, alpha, q);
                 updated[i] = Math.log10(q);
             }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java
index 429becfc7..e04bfab76 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantDataManager.java
@@ -51,10 +51,10 @@ public class VariantDataManager {
     private ExpandingArrayList data;
     private final double[] meanVector;
     private final double[] varianceVector; // this is really the standard deviation
-    public final ArrayList annotationKeys;
+    public final List annotationKeys;
     private final VariantRecalibratorArgumentCollection VRAC;
     protected final static Logger logger = Logger.getLogger(VariantDataManager.class);
-
+    protected final List trainingSets;
 
     public VariantDataManager( final List annotationKeys, final VariantRecalibratorArgumentCollection VRAC ) {
         this.data = null;
@@ -62,6 +62,7 @@ public class VariantDataManager {
         this.VRAC = VRAC;
         meanVector = new double[this.annotationKeys.size()];
         varianceVector = new double[this.annotationKeys.size()];
+        trainingSets = new ArrayList();
     }
 
     public void setData( final ExpandingArrayList data ) {
@@ -104,6 +105,31 @@ public class VariantDataManager {
         }
     }
 
+     public void addTrainingSet( final TrainingSet trainingSet ) {
+         trainingSets.add( trainingSet );
+     }
+
+     public boolean checkHasTrainingSet() {
+         for( final TrainingSet trainingSet : trainingSets ) {
+             if( trainingSet.isTraining ) { return true; }
+         }
+         return false;
+     }
+
+     public boolean checkHasTruthSet() {
+         for( final TrainingSet trainingSet : trainingSets ) {
+             if( trainingSet.isTruth ) { return true; }
+         }
+         return false;
+     }
+
+     public boolean checkHasKnownSet() {
+         for( final TrainingSet trainingSet : trainingSets ) {
+             if( trainingSet.isKnown ) { return true; }
+         }
+         return false;
+     }
+
     public ExpandingArrayList getTrainingData() {
         final ExpandingArrayList trainingData = new ExpandingArrayList();
         for( final VariantDatum datum : data ) {
@@ -232,57 +258,35 @@ public class VariantDataManager {
         return value;
     }
 
-    public void parseTrainingSets( final RefMetaDataTracker tracker, final GenomeLoc genomeLoc, final VariantContext evalVC, final VariantDatum datum, final boolean TRUST_ALL_POLYMORPHIC, final HashMap rodToPriorMap,
-                                   final List> training, final List> truth, final List> known, final List> badSites, final List> resource) {
+    public void parseTrainingSets( final RefMetaDataTracker tracker, final GenomeLoc genomeLoc, final VariantContext evalVC, final VariantDatum datum, final boolean TRUST_ALL_POLYMORPHIC ) {
         datum.isKnown = false;
         datum.atTruthSite = false;
         datum.atTrainingSite = false;
         datum.atAntiTrainingSite = false;
         datum.prior = 2.0;
 
-        //BUGBUG: need to clean this up
-
-        for( final RodBinding rod : training ) {
-            for( final VariantContext trainVC : tracker.getValues(rod, genomeLoc) ) {
+        for( final TrainingSet trainingSet : trainingSets ) {
+            for( final VariantContext trainVC : tracker.getValues(trainingSet.rodBinding, genomeLoc) ) {
                 if( isValidVariant( evalVC, trainVC, TRUST_ALL_POLYMORPHIC ) ) {
-                    datum.atTrainingSite = true;
-                    datum.prior = Math.max( datum.prior, (rodToPriorMap.containsKey(rod.getName()) ? rodToPriorMap.get(rod.getName()) : 0.0) );
+                    datum.isKnown = datum.isKnown || trainingSet.isKnown;
+                    datum.atTruthSite = datum.atTruthSite || trainingSet.isTruth;
+                    datum.atTrainingSite = datum.atTrainingSite || trainingSet.isTraining;
+                    datum.prior = Math.max( datum.prior, trainingSet.prior );
+                    datum.consensusCount += ( trainingSet.isConsensus ? 1 : 0 );
                 }
-            }
-        }
-        for( final RodBinding rod : truth ) {
-            for( final VariantContext trainVC : tracker.getValues(rod, genomeLoc) ) {
-                if( isValidVariant( evalVC, trainVC, TRUST_ALL_POLYMORPHIC ) ) {
-                    datum.atTruthSite = true;
-                    datum.prior = Math.max( datum.prior, (rodToPriorMap.containsKey(rod.getName()) ? rodToPriorMap.get(rod.getName()) : 0.0) );
-                }
-            }
-        }
-        for( final RodBinding rod : known ) {
-            for( final VariantContext trainVC : tracker.getValues(rod, genomeLoc) ) {
-                if( isValidVariant( evalVC, trainVC, TRUST_ALL_POLYMORPHIC ) ) {
-                    datum.isKnown = true;
-                    datum.prior = Math.max( datum.prior, (rodToPriorMap.containsKey(rod.getName()) ? rodToPriorMap.get(rod.getName()) : 0.0) );
-                }
-            }
-        }
-        for( final RodBinding rod : resource ) {
-            for( final VariantContext trainVC : tracker.getValues(rod, genomeLoc) ) {
-                if( isValidVariant( evalVC, trainVC, TRUST_ALL_POLYMORPHIC ) ) {
-                    datum.prior = Math.max( datum.prior, (rodToPriorMap.containsKey(rod.getName()) ? rodToPriorMap.get(rod.getName()) : 0.0) );
-                }
-            }
-        }
-        for( final RodBinding rod : badSites ) {
-            for( final VariantContext trainVC : tracker.getValues(rod, genomeLoc) ) {
                 if( trainVC != null ) {
-                    datum.atAntiTrainingSite = true;
-                    datum.prior = Math.max( datum.prior, (rodToPriorMap.containsKey(rod.getName()) ? rodToPriorMap.get(rod.getName()) : 0.0) );
+                    datum.atAntiTrainingSite = datum.atAntiTrainingSite || trainingSet.isAntiTraining;
                 }
             }
         }
     }
 
+    private boolean isValidVariant( final VariantContext evalVC, final VariantContext trainVC, final boolean TRUST_ALL_POLYMORPHIC) {
+        return trainVC != null && trainVC.isNotFiltered() && trainVC.isVariant() &&
+                        ((evalVC.isSNP() && trainVC.isSNP()) || ((evalVC.isIndel()||evalVC.isMixed()) && (trainVC.isIndel()||trainVC.isMixed()))) &&
+                        (TRUST_ALL_POLYMORPHIC || !trainVC.hasGenotypes() || trainVC.isPolymorphic());
+    }
+
     public void writeOutRecalibrationTable( final PrintStream RECAL_FILE ) {
         for( final VariantDatum datum : data ) {
             RECAL_FILE.println(String.format("%s,%d,%d,%.4f,%s",
@@ -290,10 +294,4 @@ public class VariantDataManager {
                     (datum.worstAnnotation != -1 ? annotationKeys.get(datum.worstAnnotation) : "NULL")));
         }
     }
-
-    private boolean isValidVariant( final VariantContext evalVC, final VariantContext trainVC, final boolean TRUST_ALL_POLYMORPHIC) {
-        return trainVC != null && trainVC.isNotFiltered() && trainVC.isVariant() &&
-                        ((evalVC.isSNP() && trainVC.isSNP()) || ((evalVC.isIndel()||evalVC.isMixed()) && (trainVC.isIndel()||trainVC.isMixed()))) &&
-                        (TRUST_ALL_POLYMORPHIC || !trainVC.hasGenotypes() || trainVC.isPolymorphic());
-    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java
index df4faebd1..89e702b64 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java
@@ -77,16 +77,15 @@ import java.util.*;
  * 

  * A tranches file which shows various metrics of the recalibration callset as a function of making several slices through the data.  
  *
- * 
Examples

+ * 
Example

  * 
  * java -Xmx4g -jar GenomeAnalysisTK.jar \
  *   -T VariantRecalibrator \
  *   -R reference/human_g1k_v37.fasta \
  *   -input NA12878.HiSeq.WGS.bwa.cleaned.raw.hg19.subset.vcf \
- *   -truth:prior=15.0 hapmap_3.3.b37.sites.vcf \
- *   -training:prior=15.0 hapmap_3.3.b37.sites.vcf \
- *   -training:prior=12.0 1000G_omni2.5.b37.sites.vcf \
- *   -known:prior=8.0 dbsnp_132.b37.vcf \
+ *   -resource:hapmap,known=false,training=true,truth=true,prior=15.0 hapmap_3.3.b37.sites.vcf \
+ *   -resource:omni,known=false,training=true,truth=false,prior=12.0 1000G_omni2.5.b37.sites.vcf \
+ *   -resource:dbsnp,known=true,training=false,truth=false,prior=8.0 dbsnp_132.b37.vcf \
  *   -an QD -an HaplotypeScore -an MQRankSum -an ReadPosRankSum -an FS -an MQ \
  *   -recalFile path/to/output.recal \
  *   -tranchesFile path/to/output.tranches \
@@ -112,34 +111,11 @@ public class VariantRecalibrator extends RodWalker> input;
 
     /**
-     * Input variants which are found to overlap with these training sites are used to build the Gaussian mixture model.
-     */
-    @Input(fullName="training", shortName = "training", doc="A list of training variants used to train the Gaussian mixture model", required=true)
-    public List> training;
-
-    /**
-     * When deciding where to set the cutoff in VQSLOD sensitivity to these truth sites is used.
-     * Typically one might want to say I dropped my threshold until I got back 99% of HapMap sites, for example.
-     */
-    @Input(fullName="truth", shortName = "truth", doc="A list of true variants to be used when deciding the truth sensitivity cut of the final callset", required=true)
-    public List> truth;
-
-    /**
-     * The known / novel status of a variant isn't used by the algorithm itself and is only used for reporting / display purposes.
-     * The output metrics are stratified by known status in order to aid in comparisons with other call sets.
-     */
-    @Input(fullName="known", shortName = "known", doc="A list of known variants to be used for metric comparison purposes", required=false)
-    public List> known = Collections.emptyList();
-
-    /**
-     * In addition to using the worst 3% of variants as compared to the Gaussian mixture model, we can also supplement the list
-     * with a database of known bad variants. Maybe these are loci which are frequently filtered out in many projects (centromere, for example).
-     */
-    @Input(fullName="badSites", shortName = "badSites", doc="A list of known bad variants used to supplement training the negative model", required=false)
-    public List> badSites = Collections.emptyList();
-
-    /**
-     * Any set of sites for which you would like to apply a prior probability but for which you don't want to use as training, truth, or known sites.
+     * Any set of VCF files to use as lists of training, truth, or known sites.
+     * Training - Input variants which are found to overlap with these training sites are used to build the Gaussian mixture model.
+     * Truth - When deciding where to set the cutoff in VQSLOD sensitivity to these truth sites is used.
+     * Known - The known / novel status of a variant isn't used by the algorithm itself and is only used for reporting / display purposes.
+     * Bad - In addition to using the worst 3% of variants as compared to the Gaussian mixture model, we can also supplement the list with a database of known bad variants.
      */
     @Input(fullName="resource", shortName = "resource", doc="A list of sites for which to apply a prior probability of being correct but which aren't used by the algorithm", required=false)
     public List> resource = Collections.emptyList();
@@ -179,7 +155,7 @@ public class VariantRecalibrator extends RodWalker ignoreInputFilterSet = new TreeSet();
     private final VariantRecalibratorEngine engine = new VariantRecalibratorEngine( VRAC );
-    private final HashMap rodToPriorMap = new HashMap();
 
     //---------------------------------------------------------------------------------------------------------------
     //
@@ -227,18 +202,15 @@ public class VariantRecalibrator extends RodWalker> allInputBindings = new ArrayList>();
-        allInputBindings.addAll(truth);
-        allInputBindings.addAll(training);
-        allInputBindings.addAll(known);
-        allInputBindings.addAll(badSites);
-        allInputBindings.addAll(resource);
-        for( final RodBinding rod : allInputBindings ) {
-            try {
-                rodToPriorMap.put(rod.getName(), (rod.getTags().containsKey("prior") ? Double.parseDouble(rod.getTags().getValue("prior")) : 0.0) );
-            } catch( NumberFormatException e ) {
-                throw new UserException.BadInput("Bad rod binding syntax. Prior key-value tag detected but isn't parsable. Expecting something like -training:prior=12.0 my.set.vcf");
-            }
+        for( RodBinding rod : resource ) {
+            dataManager.addTrainingSet( new TrainingSet( rod ) );
+        }
+
+        if( !dataManager.checkHasTrainingSet() ) {
+            throw new UserException.CommandLineException( "No training set found! Please provide sets of known polymorphic loci marked with the training=true ROD binding tag. For example, -B:hapmap,VCF,known=false,training=true,truth=true,prior=12.0 hapmapFile.vcf" );
+        }
+        if( !dataManager.checkHasTruthSet() ) {
+            throw new UserException.CommandLineException( "No truth set found! Please provide sets of known polymorphic loci marked with the truth=true ROD binding tag. For example, -B:hapmap,VCF,known=false,training=true,truth=true,prior=12.0 hapmapFile.vcf" );
         }
     }
 
@@ -270,7 +242,7 @@ public class VariantRecalibrator extends RodWalker {
 
         if (minimumN > 1 && (vcs.size() - numFilteredRecords < minimumN))
             return 0;
-        
+
         List mergedVCs = new ArrayList();
         Map> VCsByType = VariantContextUtils.separateVariantContextsByType(vcs);
         // iterate over the types so that it's deterministic
@@ -244,7 +244,7 @@ public class CombineVariants extends RodWalker {
                         SET_KEY, filteredAreUncalled, MERGE_INFO_WITH_MAX_AC));
         }
 
-        for ( VariantContext mergedVC : mergedVCs ) {
+         for ( VariantContext mergedVC : mergedVCs ) {
             // only operate at the start of events
             if ( mergedVC == null )
                 continue;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/LiftoverVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/LiftoverVariants.java
index 1c76a21ea..a932d44ed 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/LiftoverVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/LiftoverVariants.java
@@ -99,7 +99,7 @@ public class LiftoverVariants extends RodWalker {
 
 
         final VCFHeader vcfHeader = new VCFHeader(metaData, samples);
-        writer = new StandardVCFWriter(file, false);
+        writer = new StandardVCFWriter(file, getMasterSequenceDictionary(), false);
         writer.writeHeader(vcfHeader);
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/RandomlySplitVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/RandomlySplitVariants.java
index 1fefd20fc..fa5093839 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/RandomlySplitVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/RandomlySplitVariants.java
@@ -75,7 +75,7 @@ public class RandomlySplitVariants extends RodWalker {
         hInfo.addAll(VCFUtils.getHeaderFields(getToolkit(), inputNames));
 
         vcfWriter1.writeHeader(new VCFHeader(hInfo, samples));
-        vcfWriter2 = new StandardVCFWriter(file2, true);
+        vcfWriter2 = new StandardVCFWriter(file2, getMasterSequenceDictionary(), true);
         vcfWriter2.writeHeader(new VCFHeader(hInfo, samples));
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
index 35ff66243..bf9a3b03e 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariants.java
@@ -145,10 +145,9 @@ import java.util.*;
  *   -R ref.fasta \
  *   -T SelectVariants \
  *   --variant input.vcf \
- *   -o output.vcf \
- *   -SM family.yaml \
  *   -family NA12891+NA12892=NA12878 \
- *   -mvq 50
+ *   -mvq 50 \
+ *   -o violations.vcf
  *
  * Creating a sample of exactly 1000 variants randomly chosen with equal probability from the variant VCF:
  * java -Xmx2g -jar GenomeAnalysisTK.jar \
@@ -265,17 +264,17 @@ public class SelectVariants extends RodWalker {
     private File AF_FILE = new File("");
 
     @Hidden
-    @Argument(fullName="family_structure_file", shortName="familyFile", doc="USE YAML FILE INSTEAD (-SM) !!! string formatted as dad+mom=child where these parameters determine which sample names are examined", required=false)
+    @Argument(fullName="family_structure_file", shortName="familyFile", doc="use -family unless you know what you're doing", required=false)
     private File FAMILY_STRUCTURE_FILE = null;
 
     /**
      * String formatted as dad+mom=child where these parameters determine which sample names are examined.
      */
-    @Argument(fullName="family_structure", shortName="family", doc="Deprecated; use the -SM argument instead", required=false)
+    @Argument(fullName="family_structure", shortName="family", doc="string formatted as dad+mom=child where these parameters determine which sample names are examined", required=false)
     private String FAMILY_STRUCTURE = "";
 
     /**
-     * Sample metadata information will be taken from a YAML file (see the -SM argument).
+     * This activates the mendelian violation module that will select all variants that correspond to a mendelian violation following the rules given by the family structure.
      */
     @Argument(fullName="mendelianViolation", shortName="mv", doc="output mendelian violation sites only", required=false)
     private Boolean MENDELIAN_VIOLATIONS = false;
@@ -306,7 +305,7 @@ public class SelectVariants extends RodWalker {
 
 
     @Hidden
-    @Argument(fullName="outMVFile", shortName="outMVFile", doc="USE YAML FILE INSTEAD (-SM) !!! string formatted as dad+mom=child where these parameters determine which sample names are examined", required=false)
+    @Argument(fullName="outMVFile", shortName="outMVFile", doc="", required=false)
     private String outMVFile = null;
 
     /* Private class used to store the intermediate variants in the integer random selection process */
@@ -453,7 +452,7 @@ public class SelectVariants extends RodWalker {
                         throw new UserException.CouldNotCreateOutputFile(outMVFile, "Can't open output file", e);   }
             }
             else
-                mvSet.add(new MendelianViolation(getToolkit(), MENDELIAN_VIOLATION_QUAL_THRESHOLD));
+                mvSet.add(new MendelianViolation(FAMILY_STRUCTURE, MENDELIAN_VIOLATION_QUAL_THRESHOLD));
         }
         else if (!FAMILY_STRUCTURE.isEmpty()) {
             mvSet.add(new MendelianViolation(FAMILY_STRUCTURE, MENDELIAN_VIOLATION_QUAL_THRESHOLD));
@@ -575,7 +574,7 @@ public class SelectVariants extends RodWalker {
                         // ok we have a comp VC and we need to match the AF spectrum of inputAFRodName.
                         // We then pick a variant with probablity AF*desiredFraction
                         if ( sub.hasAttribute(VCFConstants.ALLELE_FREQUENCY_KEY) )  {
-                            String afo = sub.getAttributeAsString(VCFConstants.ALLELE_FREQUENCY_KEY);
+                            String afo = sub.getAttributeAsString(VCFConstants.ALLELE_FREQUENCY_KEY, null);
 
                             double af;
                             double afBoost = 1.0;
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
index 2c7902914..fdfca982c 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariants.java
@@ -56,11 +56,6 @@ import java.util.Set;
  * A variant set to filter.
  * 

  *
- * 
Output

- * 

- * A filtered VCF.
- * 

- *
  * 
Examples

  * 
  * java -Xmx2g -jar GenomeAnalysisTK.jar \
diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantValidationAssessor.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantValidationAssessor.java
index b98646270..8eaf976d0 100644
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantValidationAssessor.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantValidationAssessor.java
@@ -41,7 +41,7 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContextUtils;
 import java.util.*;
 
 /**
- * Annotates a validation (from e.g. Sequenom) VCF with QC metrics (HW-equilibrium, % failed probes)
+ * Annotates a validation (from Sequenom for example) VCF with QC metrics (HW-equilibrium, % failed probes)
  *
  * 

  * The Variant Validation Assessor is a tool for vetting/assessing validation data (containing genotypes).
@@ -57,7 +57,16 @@ import java.util.*;
  *
  * 
Output

  * 

- * An annotated VCF.
+ * An annotated VCF.  Additionally, a table like the following will be output:
+ * 
+ *     Total number of samples assayed:                  185
+ *     Total number of records processed:                152
+ *     Number of Hardy-Weinberg violations:              34 (22%)
+ *     Number of no-call violations:                     12 (7%)
+ *     Number of homozygous variant violations:          0 (0%)
+ *     Number of records passing all filters:            106 (69%)
+ *     Number of passing records that are polymorphic:   98 (92%)
+ * 

  * 

  *
  * 
Examples

diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java
index 2a877fb09..c44d84136 100755
--- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java
+++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantutils/VariantsToTable.java
@@ -192,7 +192,7 @@ public class VariantsToTable extends RodWalker {
             if ( getters.containsKey(field) ) {
                 val = getters.get(field).get(vc);
             } else if ( vc.hasAttribute(field) ) {
-                val = vc.getAttributeAsString(field);
+                val = vc.getAttributeAsString(field, null);
             } else if ( isWildCard(field) ) {
                 Set wildVals = new HashSet();
                 for ( Map.Entry elt : vc.getAttributes().entrySet()) {
@@ -309,6 +309,7 @@ public class VariantsToTable extends RodWalker {
         getters.put("HOM-REF", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHomRefCount()); } });
         getters.put("HOM-VAR", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHomVarCount()); } });
         getters.put("NO-CALL", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getNoCallCount()); } });
+        getters.put("TYPE", new Getter() { public String get(VariantContext vc) { return vc.getType().toString(); } });
         getters.put("VAR", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getHetCount() + vc.getHomVarCount()); } });
         getters.put("NSAMPLES", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getNSamples()); } });
         getters.put("NCALLED", new Getter() { public String get(VariantContext vc) { return Integer.toString(vc.getNSamples() - vc.getNoCallCount()); } });
diff --git a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java
index b96923589..b66198713 100644
--- a/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java
+++ b/public/java/src/org/broadinstitute/sting/utils/GenomeLoc.java
@@ -306,7 +306,7 @@ public class GenomeLoc implements Comparable, Serializable, HasGenome
     
     @Override
     public int hashCode() {
-        return (int)( start << 16 + stop << 4 + contigIndex );
+        return start << 16 | stop << 4 | contigIndex;
     }
 
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/genotype/Haplotype.java b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java
similarity index 98%
rename from public/java/src/org/broadinstitute/sting/utils/genotype/Haplotype.java
rename to public/java/src/org/broadinstitute/sting/utils/Haplotype.java
index a17e81461..ce2ca2c28 100755
--- a/public/java/src/org/broadinstitute/sting/utils/genotype/Haplotype.java
+++ b/public/java/src/org/broadinstitute/sting/utils/Haplotype.java
@@ -22,10 +22,9 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package org.broadinstitute.sting.utils.genotype;
+package org.broadinstitute.sting.utils;
 
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
-import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.variantcontext.Allele;
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java
index f4c057c15..17e74c4f1 100644
--- a/public/java/src/org/broadinstitute/sting/utils/MathUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/MathUtils.java
@@ -444,11 +444,25 @@ public class MathUtils {
      * @return a newly allocated array corresponding the normalized values in array, maybe log10 transformed
     */
     public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOutput) {
-        double[] normalized = new double[array.length];
+        return normalizeFromLog10(array, takeLog10OfOutput, false);
+    }
+
+    public static double[] normalizeFromLog10(double[] array, boolean takeLog10OfOutput, boolean keepInLogSpace) {
 
         // for precision purposes, we need to add (or really subtract, since they're
         // all negative) the largest value; also, we need to convert to normal-space.
         double maxValue = Utils.findMaxEntry(array);
+
+        // we may decide to just normalize in log space with converting to linear space
+        if (keepInLogSpace) {
+            for (int i = 0; i < array.length; i++)
+                array[i] -= maxValue;
+            return array;
+        }
+
+        // default case: go to linear space
+        double[] normalized = new double[array.length];
+
         for (int i = 0; i < array.length; i++)
             normalized[i] = Math.pow(10, array[i] - maxValue);
 
@@ -1056,42 +1070,30 @@ public class MathUtils {
     }
 
     static public double softMax(final double x, final double y) {
-         if (Double.isInfinite(x))
-             return y;
+        // we need to compute log10(10^x + 10^y)
+        // By Jacobian logarithm identity, this is equal to
+        // max(x,y) + log10(1+10^-abs(x-y))
+        // we compute the second term as a table lookup
+        // with integer quantization
 
-         if (Double.isInfinite(y))
-             return x;
+        // slow exact version:
+        // return Math.log10(Math.pow(10.0,x) + Math.pow(10.0,y));
 
-         if (y >= x + MAX_JACOBIAN_TOLERANCE)
-             return y;
-         if (x >= y + MAX_JACOBIAN_TOLERANCE)
-             return x;
+        double diff = x-y;
 
-         // OK, so |y-x| < tol: we use the following identity then:
-         // we need to compute log10(10^x + 10^y)
-         // By Jacobian logarithm identity, this is equal to
-         // max(x,y) + log10(1+10^-abs(x-y))
-         // we compute the second term as a table lookup
-         // with integer quantization
-
-         //double diff = Math.abs(x-y);
-         double diff = x-y;
-         double t1 =x;
-         if (diff<0) { //
-             t1 = y;
-             diff= -diff;
-         }
-         // t has max(x,y), diff has abs(x-y)
-         // we have pre-stored correction for 0,0.1,0.2,... 10.0
-         //int ind = (int)Math.round(diff*INV_JACOBIAN_LOG_TABLE_STEP);
-         int ind = (int)(diff*INV_JACOBIAN_LOG_TABLE_STEP+0.5);
-         // gdebug+
-         //double z =Math.log10(1+Math.pow(10.0,-diff));
-         //System.out.format("x: %f, y:%f, app: %f, true: %f ind:%d\n",x,y,t2,z,ind);
-         //gdebug-
-         return t1+jacobianLogTable[ind];
-         // return Math.log10(Math.pow(10.0,x) + Math.pow(10.0,y));
-     }
+        if (diff > MAX_JACOBIAN_TOLERANCE)
+            return x;
+        else if (diff < -MAX_JACOBIAN_TOLERANCE)
+            return y;
+        else if (diff >= 0) {
+            int ind = (int)(diff*INV_JACOBIAN_LOG_TABLE_STEP+0.5);
+            return x + jacobianLogTable[ind];
+        }
+        else {
+            int ind = (int)(-diff*INV_JACOBIAN_LOG_TABLE_STEP+0.5);
+            return y + jacobianLogTable[ind];
+        }
+    }
 
     public static double phredScaleToProbability (byte q) {
         return Math.pow(10,(-q)/10.0);
diff --git a/public/java/src/org/broadinstitute/sting/utils/MendelianViolation.java b/public/java/src/org/broadinstitute/sting/utils/MendelianViolation.java
index c6a07b5ce..8da118174 100755
--- a/public/java/src/org/broadinstitute/sting/utils/MendelianViolation.java
+++ b/public/java/src/org/broadinstitute/sting/utils/MendelianViolation.java
@@ -1,11 +1,13 @@
 package org.broadinstitute.sting.utils;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
 import org.broadinstitute.sting.gatk.datasources.sample.Sample;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.variantcontext.Genotype;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
+import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -32,6 +34,9 @@ public class MendelianViolation {
 
     private static Pattern FAMILY_PATTERN = Pattern.compile("(.*)\\+(.*)=(.*)");
 
+    static final int[] mvOffsets = new int[] { 1,2,5,6,8,11,15,18,20,21,24,25 };
+    static final int[] nonMVOffsets = new int[]{ 0,3,4,7,9,10,12,13,14,16,17,19,22,23,26 };
+
 
     public String getSampleMom() {
         return sampleMom;
@@ -168,4 +173,41 @@ public class MendelianViolation {
         return true;
     }
 
+    /**
+     * @return the likelihood ratio for a mendelian violation
+     */
+    public double violationLikelihoodRatio(VariantContext vc) {
+        double[] logLikAssignments = new double[27];
+        // the matrix to set up is
+        // MOM   DAD    CHILD
+        //                    |-  AA
+        //   AA     AA    |    AB
+        //                    |-   BB
+        //                    |- AA
+        //  AA     AB     |   AB
+        //                    |- BB
+        // etc. The leaves are counted as 0-11 for MVs and 0-14 for non-MVs
+        double[] momGL = vc.getGenotype(sampleMom).getLikelihoods().getAsVector();
+        double[] dadGL = vc.getGenotype(sampleDad).getLikelihoods().getAsVector();
+        double[] childGL = vc.getGenotype(sampleChild).getLikelihoods().getAsVector();
+        int offset = 0;
+        for ( int oMom = 0; oMom < 3; oMom++ ) {
+            for ( int oDad = 0; oDad < 3; oDad++ ) {
+                for ( int oChild = 0; oChild < 3; oChild ++ ) {
+                    logLikAssignments[offset++] = momGL[oMom] + dadGL[oDad] + childGL[oChild];
+                }
+            }
+        }
+        double[] mvLiks = new double[12];
+        double[] nonMVLiks = new double[15];
+        for ( int i = 0; i < 12; i ++ ) {
+            mvLiks[i] = logLikAssignments[mvOffsets[i]];
+        }
+
+        for ( int i = 0; i < 15; i++) {
+            nonMVLiks[i] = logLikAssignments[nonMVOffsets[i]];
+        }
+
+        return MathUtils.log10sumLog10(mvLiks) - MathUtils.log10sumLog10(nonMVLiks);
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java b/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java
index fad2320fc..19e03a19d 100755
--- a/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/QualityUtils.java
@@ -9,14 +9,17 @@ import net.sf.samtools.SAMUtils;
  * @author Kiran Garimella
  */
 public class QualityUtils {
-
     public final static byte MAX_QUAL_SCORE = SAMUtils.MAX_PHRED_SCORE;
     public final static double MIN_REASONABLE_ERROR = 0.0001;
     public final static byte MAX_REASONABLE_Q_SCORE = 40;
     public final static byte MIN_USABLE_Q_SCORE = 6;
-
     public final static int MAPPING_QUALITY_UNAVAILABLE = 255;
 
+    private static double qualToErrorProbCache[] = new double[256];
+    static {
+        for (int i = 0; i < 256; i++) qualToErrorProbCache[i] = qualToErrorProbRaw((byte)i);
+    }
+
     /**
      * Private constructor.  No instantiating this class!
      */
@@ -33,10 +36,6 @@ public class QualityUtils {
         return 1.0 - qualToErrorProb(qual);
     }
 
-    static public double qualToProb(int qual) {
-        return qualToProb( (double)qual );
-    }
-
     static public double qualToProb(double qual) {
         return 1.0 - Math.pow(10.0, qual/(-10.0));
     }
@@ -48,10 +47,14 @@ public class QualityUtils {
      * @param qual a quality score (0-40)
      * @return a probability (0.0-1.0)
      */
-    static public double qualToErrorProb(byte qual) {
+    static public double qualToErrorProbRaw(byte qual) {
         return Math.pow(10.0, ((double) qual)/-10.0);
     }
 
+    static public double qualToErrorProb(byte qual) {
+        return qualToErrorProbCache[qual];
+    }
+
     /**
      * Convert a probability to a quality score.  Note, this is capped at Q40.
      *
@@ -110,88 +113,4 @@ public class QualityUtils {
         //return (byte) Math.min(qual, maxQual);
         return (byte) Math.max(Math.min(qual, maxQual), 1);
     }
-
-    /**
-     * Compress a base and a probability into a single byte so that it can be output in a SAMRecord's SQ field.
-     * Note: the highest probability this function can encode is 64%, so this function should only never be used on the best base hypothesis.
-     * Another note: the probability encoded here gets rounded to the nearest 1%.
-     *
-     * @param baseIndex the base index
-     * @param prob      the base probability
-     * @return a byte containing the index and the probability
-     */
-    static public byte baseAndProbToCompressedQuality(int baseIndex, double prob) {
-        byte compressedQual = 0;
-
-        compressedQual = (byte) baseIndex;
-
-        byte cprob = (byte) (100.0*prob);
-        byte qualmask = (byte) 252;
-        compressedQual += ((cprob << 2) & qualmask);
-        
-        return compressedQual;
-    }
-
-    /**
-     * From a compressed base, extract the base index (0:A, 1:C, 2:G, 3:T)
-     *
-     * @param compressedQual the compressed quality score, as returned by baseAndProbToCompressedQuality
-     * @return base index
-     */
-    static public int compressedQualityToBaseIndex(byte compressedQual) {
-        return (int) (compressedQual & 0x3);
-    }
-
-    /**
-     * From a compressed base, extract the base probability
-     *
-     * @param compressedQual the compressed quality score, as returned by baseAndProbToCompressedQuality
-     * @return the probability
-     */
-    static public double compressedQualityToProb(byte compressedQual) {
-        // Because java natives are signed, extra care must be taken to avoid
-        // shifting a 1 into the sign bit in the implicit promotion of 2 to an int.
-        int x2 = ((int) compressedQual) & 0xff;
-        x2 = (x2 >>> 2);
-
-        return ((double) x2)/100.0;
-    }
-
-    /**
-     * Return the complement of a compressed quality
-     *
-     * @param compressedQual  the compressed quality score (as returned by baseAndProbToCompressedQuality)
-     * @return the complementary compressed quality
-     */
-    static public byte complementCompressedQuality(byte compressedQual) {
-        int baseIndex = compressedQualityToBaseIndex(compressedQual);
-        double prob = compressedQualityToProb(compressedQual);
-
-        return baseAndProbToCompressedQuality(BaseUtils.complementIndex(baseIndex), prob);
-    }
-
-    /**
-     * Return the reverse complement of a byte array of compressed qualities
-     *
-     * @param compressedQuals  a byte array of compressed quality scores
-     * @return the reverse complement of the byte array
-     */
-    static public byte[] reverseComplementCompressedQualityArray(byte[] compressedQuals) {
-        byte[] rcCompressedQuals = new byte[compressedQuals.length];
-
-        for (int pos = 0; pos < compressedQuals.length; pos++) {
-            rcCompressedQuals[compressedQuals.length - pos - 1] = complementCompressedQuality(compressedQuals[pos]);
-        }
-
-        return rcCompressedQuals;
-    }
-
-    /**
-     * Return the reverse of a byte array of qualities (compressed or otherwise)
-     * @param quals   the array of bytes to be reversed
-     * @return the reverse of the quality array
-     */
-    static public byte[] reverseQualityArray( byte[] quals ) {
-        return Utils.reverse(quals); // no sense in duplicating functionality
-    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java b/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java
index c0493fe22..58f7942fe 100644
--- a/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java
+++ b/public/java/src/org/broadinstitute/sting/utils/R/RScriptExecutor.java
@@ -53,7 +53,7 @@ public class RScriptExecutor {
 
     public static class RScriptArgumentCollection {
         @Advanced
-        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/tools/apps/R-2.6.0/bin/Rscript", required = false)
+        @Argument(fullName = "path_to_Rscript", shortName = "Rscript", doc = "The path to your implementation of Rscript. For Broad users this is maybe /broad/software/free/Linux/redhat_5_x86_64/pkgs/r_2.12.0/bin/Rscript", required = false)
         public String PATH_TO_RSCRIPT = "Rscript";
 
         @Advanced
diff --git a/public/java/src/org/broadinstitute/sting/utils/SampleUtils.java b/public/java/src/org/broadinstitute/sting/utils/SampleUtils.java
index f9997bfd8..1b4703e4a 100755
--- a/public/java/src/org/broadinstitute/sting/utils/SampleUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/SampleUtils.java
@@ -190,11 +190,21 @@ public class SampleUtils {
 
     }
 
-    public static List getSamplesFromCommandLineInput(Collection sampleArgs) {
+    /**
+     * Returns a new set of samples, containing a final list of samples expanded from sampleArgs
+     *
+     * Each element E of sampleArgs can either be a literal sample name or a file.  For each E,
+     * we try to read a file named E from disk, and if possible all lines from that file are expanded
+     * into unique sample names.
+     *
+     * @param sampleArgs
+     * @return
+     */
+    public static Set getSamplesFromCommandLineInput(Collection sampleArgs) {
         if (sampleArgs != null) {
             // Let's first go through the list and see if we were given any files.  We'll add every entry in the file to our
             // sample list set, and treat the entries as if they had been specified on the command line.
-            List samplesFromFiles = new ArrayList();
+            Set samplesFromFiles = new HashSet();
             for (String SAMPLE_EXPRESSION : sampleArgs) {
                 File sampleFile = new File(SAMPLE_EXPRESSION);
 
@@ -203,7 +213,7 @@ public class SampleUtils {
 
                     List lines = reader.readLines();
                     for (String line : lines) {
-                        samplesFromFiles.add(line);
+                        samplesFromFiles.add(line.trim());
                     }
                 } catch (FileNotFoundException e) {
                     samplesFromFiles.add(SAMPLE_EXPRESSION); // not a file, so must be a sample
@@ -212,7 +222,8 @@ public class SampleUtils {
 
             return samplesFromFiles;
         }
-        return new ArrayList();
+
+        return new HashSet();
     }
 
     public static Set getSamplesFromCommandLineInput(Collection vcfSamples, Collection sampleExpressions) {
diff --git a/public/java/src/org/broadinstitute/sting/utils/Utils.java b/public/java/src/org/broadinstitute/sting/utils/Utils.java
index f6edb319f..6ce492c63 100755
--- a/public/java/src/org/broadinstitute/sting/utils/Utils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/Utils.java
@@ -240,22 +240,34 @@ public class Utils {
         return ret.toString();
     }
 
-    //public static String join(String separator, Collection strings) {
-    //    return join( separator, strings.toArray(new String[0]) );
-    //}
-
-    public static  String join(String separator, Collection objects) {
-        if(objects.isEmpty()) {
+    /**
+     * Returns a string of the form elt1.toString() [sep elt2.toString() ... sep elt.toString()] for a collection of
+     * elti objects (note there's no actual space between sep and the elti elements).  Returns
+     * "" if collection is empty.  If collection contains just elt, then returns elt.toString()
+     *
+     * @param separator the string to use to separate objects
+     * @param objects a collection of objects.  the element order is defined by the iterator over objects
+     * @param  the type of the objects
+     * @return a non-null string
+     */
+    public static  String join(final String separator, final Collection objects) {
+        if (objects.isEmpty()) { // fast path for empty collection
             return "";
-        }
-        Iterator iter = objects.iterator();
-        final StringBuilder ret = new StringBuilder(iter.next().toString());
-        while(iter.hasNext()) {
-            ret.append(separator);
-            ret.append(iter.next().toString());
-        }
+        } else {
+            final Iterator iter = objects.iterator();
+            final T first = iter.next();
 
-        return ret.toString();
+            if ( ! iter.hasNext() ) // fast path for singleton collections
+                return first.toString();
+            else { // full path for 2+ collection that actually need a join
+                final StringBuilder ret = new StringBuilder(first.toString());
+                while(iter.hasNext()) {
+                    ret.append(separator);
+                    ret.append(iter.next().toString());
+                }
+                return ret.toString();
+            }
+        }
     }
 
     public static String dupString(char c, int nCopies) {
diff --git a/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java b/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java
index ef7cf751e..4f096f86e 100644
--- a/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java
+++ b/public/java/src/org/broadinstitute/sting/utils/baq/BAQ.java
@@ -5,6 +5,7 @@ import net.sf.picard.reference.ReferenceSequence;
 import net.sf.samtools.CigarElement;
 import net.sf.samtools.CigarOperator;
 import net.sf.samtools.SAMRecord;
+import net.sf.samtools.SAMUtils;
 import org.broadinstitute.sting.utils.collections.Pair;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -131,19 +132,18 @@ public class BAQ {
     private final static double EM = 0.33333333333;
     private final static double EI = 0.25;
 
-    private double[][][] EPSILONS = new double[256][256][64];
+    private double[][][] EPSILONS = new double[256][256][SAMUtils.MAX_PHRED_SCORE+1];
 
     private void initializeCachedData() {
         for ( int i = 0; i < 256; i++ )
             for ( int j = 0; j < 256; j++ )
-                for ( int q = 0; q < 64; q++ ) {
-                    double qual = qual2prob[q < minBaseQual ? minBaseQual : q];
+                for ( int q = 0; q <= SAMUtils.MAX_PHRED_SCORE; q++ ) {
                     EPSILONS[i][j][q] = 1.0;
                 }
 
         for ( char b1 : "ACGTacgt".toCharArray() ) {
             for ( char b2 : "ACGTacgt".toCharArray() ) {
-                for ( int q = 0; q < 64; q++ ) {
+                for ( int q = 0; q <= SAMUtils.MAX_PHRED_SCORE; q++ ) {
                     double qual = qual2prob[q < minBaseQual ? minBaseQual : q];
                     double e = Character.toLowerCase(b1) == Character.toLowerCase(b2) ? 1 - qual : qual * EM;
                     EPSILONS[(byte)b1][(byte)b2][q] = e;
@@ -152,7 +152,7 @@ public class BAQ {
         }
     }
 
-    private double calcEpsilon( byte ref, byte read, byte qualB ) {
+    protected double calcEpsilon( byte ref, byte read, byte qualB ) {
         return EPSILONS[ref][read][qualB];
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingOp.java b/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingOp.java
index bc200372f..81d00d9d7 100644
--- a/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingOp.java
+++ b/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingOp.java
@@ -324,6 +324,8 @@ public class ClippingOp {
 
                 if (index <= stop && cigarElementIterator.hasNext())
                     cigarElement = cigarElementIterator.next();
+                else
+                    break;
             }
 
             // add the remaining cigar elements
@@ -363,6 +365,8 @@ public class ClippingOp {
                 index += shift;
                 if (index < start && cigarElementIterator.hasNext())
                     cigarElement = cigarElementIterator.next();
+                else
+                    break;
             }
 
             // check if we are hard clipping indels
@@ -394,7 +398,9 @@ public class ClippingOp {
 
         for (int i = 1; i <= 2; i++) {
             int shift = 0;
+            int totalHardClip = 0;
             boolean readHasStarted = false;
+            boolean addedHardClips = false;
 
             while(!cigarStack.empty()) {
                 CigarElement cigarElement = cigarStack.pop();
@@ -404,14 +410,33 @@ public class ClippingOp {
                         cigarElement.getOperator() != CigarOperator.DELETION &&
                         cigarElement.getOperator() != CigarOperator.HARD_CLIP)
                     readHasStarted = true;
+
+                else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.HARD_CLIP)
+                    totalHardClip += cigarElement.getLength();
+
                 else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.INSERTION)
                     shift += cigarElement.getLength();
 
-                if (readHasStarted || cigarElement.getOperator() == CigarOperator.HARD_CLIP) {
-                    if (i==1)
+                else if ( !readHasStarted && cigarElement.getOperator() == CigarOperator.DELETION)
+                    totalHardClip += cigarElement.getLength();
+
+                if (readHasStarted) {
+                    if (i==1) {
+                        if (!addedHardClips) {
+                            if (totalHardClip > 0)
+                                inverseCigarStack.push(new CigarElement(totalHardClip, CigarOperator.HARD_CLIP));
+                            addedHardClips = true;
+                        }
                         inverseCigarStack.push(cigarElement);
-                    else
+                    }
+                    else {
+                        if (!addedHardClips) {
+                            if (totalHardClip > 0)
+                                cleanCigar.add(new CigarElement(totalHardClip, CigarOperator.HARD_CLIP));
+                            addedHardClips = true;
+                        }
                         cleanCigar.add(cigarElement);
+                    }
                 }
             }
             // first pass  (i=1) is from end to start of the cigar elements
@@ -428,39 +453,35 @@ public class ClippingOp {
     }
 
     private int calculateAlignmentStartShift(Cigar oldCigar, Cigar newCigar) {
-        int shift = 0;
+        int newShift = 0;
+        int oldShift = 0;
 
-        // Rewind to previous start (by counting everything that was already clipped in this read)
-        for (CigarElement cigarElement : oldCigar.getCigarElements()) {
-            if (!cigarElement.getOperator().consumesReferenceBases())
-                shift -= cigarElement.getLength();
-            else
-                break;
-        }
-
-        // Advance to new start (by counting everything new that has been clipped )
         for (CigarElement cigarElement : newCigar.getCigarElements()) {
-            if (!cigarElement.getOperator().consumesReferenceBases())
-                shift += cigarElement.getLength();
+            if (cigarElement.getOperator() == CigarOperator.HARD_CLIP || cigarElement.getOperator() == CigarOperator.SOFT_CLIP)
+                newShift += cigarElement.getLength();
             else
                 break;
         }
 
-        return shift;
+        for (CigarElement cigarElement : oldCigar.getCigarElements()) {
+            if (cigarElement.getOperator() == CigarOperator.HARD_CLIP || cigarElement.getOperator() == CigarOperator.SOFT_CLIP )
+                oldShift += Math.min(cigarElement.getLength(), newShift - oldShift);
+            else
+                break;
+        }
+        return newShift - oldShift;
     }
 
     private int calculateHardClippingAlignmentShift(CigarElement cigarElement, int clippedLength) {
-        if (cigarElement.getOperator() == CigarOperator.INSERTION) {
-            int cigarElementLength = cigarElement.getLength();
-            if (clippedLength >= cigarElementLength)
-                return -cigarElement.getLength();
-            else
-                return -clippedLength;
-        }
+        // Insertions should be discounted from the total hard clip count
+        if (cigarElement.getOperator() == CigarOperator.INSERTION)
+            return -clippedLength;
 
-        if (cigarElement.getOperator() == CigarOperator.DELETION)
+        // Deletions should be added to the total hard clip count
+        else if (cigarElement.getOperator() == CigarOperator.DELETION)
             return cigarElement.getLength();
 
+        // There is no shift if we are not clipping an indel
         return 0;
     }
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingRepresentation.java b/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingRepresentation.java
index 0dbe55726..d574ba2f0 100644
--- a/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingRepresentation.java
+++ b/public/java/src/org/broadinstitute/sting/utils/clipreads/ClippingRepresentation.java
@@ -21,6 +21,8 @@ public enum ClippingRepresentation {
     SOFTCLIP_BASES,
 
     /**
+     * WARNING: THIS OPTION IS STILL UNDER DEVELOPMENT AND IS NOT SUPPORTED.
+     *
      * Change the read's cigar string to hard clip (H, see sam-spec) away the bases.
      * Hard clipping, unlike soft clipping, actually removes bases from the read,
      * reducing the resulting file's size but introducing an irrevesible (i.e.,
diff --git a/public/java/src/org/broadinstitute/sting/utils/clipreads/ReadClipper.java b/public/java/src/org/broadinstitute/sting/utils/clipreads/ReadClipper.java
index 26c25850a..3b83617cf 100644
--- a/public/java/src/org/broadinstitute/sting/utils/clipreads/ReadClipper.java
+++ b/public/java/src/org/broadinstitute/sting/utils/clipreads/ReadClipper.java
@@ -1,6 +1,8 @@
 package org.broadinstitute.sting.utils.clipreads;
 
 import com.google.java.contract.Requires;
+import net.sf.samtools.CigarElement;
+import net.sf.samtools.CigarOperator;
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
@@ -56,6 +58,15 @@ public class ReadClipper {
         return hardClipByReferenceCoordinates(refStart, -1);
     }
 
+    private int numDeletions(SAMRecord read) {
+        int result = 0;
+        for (CigarElement e: read.getCigar().getCigarElements()) {
+            if ( e.getOperator() == CigarOperator.DELETION || e.getOperator() == CigarOperator.D )
+                result =+ e.getLength();
+        }
+        return result;
+    }
+
     private SAMRecord hardClipByReferenceCoordinates(int refStart, int refStop) {
         int start = (refStart < 0) ? 0 : ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStart);
         int stop =  (refStop  < 0) ? read.getReadLength() - 1 : ReadUtils.getReadCoordinateForReferenceCoordinate(read, refStop);
@@ -63,9 +74,17 @@ public class ReadClipper {
         if (start < 0 || stop > read.getReadLength() - 1)
             throw new ReviewedStingException("Trying to clip before the start or after the end of a read");
 
-        // TODO add requires statement/check in the Hardclip function
-        if ( start > stop )
-            stop = ReadUtils.getReadCoordinateForReferenceCoordinate(read, ReadUtils.getRefCoordSoftUnclippedEnd(read));
+        if ( start > stop ) {
+//            stop = ReadUtils.getReadCoordinateForReferenceCoordinate(read, ReadUtils.getRefCoordSoftUnclippedEnd(read));
+            throw new ReviewedStingException("START > STOP -- this should never happen -- call Mauricio!");
+        }
+
+        //This tries to fix the bug where the deletion is counted a read base and as a result, the hardCLipper runs into
+        //an endless loop when hard clipping the cigar string because the read coordinates are not covered by the read
+//        stop -= numDeletions(read);
+//        if ( start > stop )
+//            start -= numDeletions(read);
+
 
         //System.out.println("Clipping start/stop: " + start + "/" + stop);
         this.addOp(new ClippingOp(start, stop));
@@ -109,6 +128,39 @@ public class ReadClipper {
         return this.clipRead(ClippingRepresentation.HARDCLIP_BASES);
     }
 
+    public SAMRecord hardClipSoftClippedBases () {
+        int readIndex = 0;
+        int cutLeft = -1;            // first position to hard clip (inclusive)
+        int cutRight = -1;           // first position to hard clip (inclusive)
+        boolean rightTail = false;   // trigger to stop clipping the left tail and start cutting the right tail
+
+        for (CigarElement cigarElement : read.getCigar().getCigarElements()) {
+            if (cigarElement.getOperator() == CigarOperator.SOFT_CLIP) {
+                if (rightTail) {
+                    cutRight = readIndex;
+                }
+                else {
+                    cutLeft = readIndex + cigarElement.getLength() - 1;
+                }
+            }
+            else if (cigarElement.getOperator() != CigarOperator.HARD_CLIP)
+                rightTail = true;
+
+            if (cigarElement.getOperator().consumesReadBases())
+                readIndex += cigarElement.getLength();
+        }
+
+        // It is extremely important that we cut the end first otherwise the read coordinates change.
+        if (cutRight >= 0)
+            this.addOp(new ClippingOp(cutRight, read.getReadLength() - 1));
+        if (cutLeft >= 0)
+            this.addOp(new ClippingOp(0, cutLeft));
+
+        return clipRead(ClippingRepresentation.HARDCLIP_BASES);
+    }
+
+
+
     /**
      * Return a new read corresponding to this.read that's been clipped according to ops, if any are present.
      *
@@ -131,4 +183,21 @@ public class ReadClipper {
             }
         }
     }
+
+    public SAMRecord hardClipLeadingInsertions() {
+        for(CigarElement cigarElement : read.getCigar().getCigarElements()) {
+            if (cigarElement.getOperator() != CigarOperator.HARD_CLIP && cigarElement.getOperator() != CigarOperator.SOFT_CLIP &&
+                cigarElement.getOperator() != CigarOperator.INSERTION && cigarElement.getOperator() != CigarOperator.DELETION)
+                break;
+
+            else if (cigarElement.getOperator() == CigarOperator.INSERTION) {
+                this.addOp(new ClippingOp(0, cigarElement.getLength() - 1));
+            }
+
+            else if (cigarElement.getOperator() == CigarOperator.DELETION) {
+                throw new ReviewedStingException("No read should start with a deletion. Aligner bug?");
+            }
+        }
+        return clipRead(ClippingRepresentation.HARDCLIP_BASES);
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/refseq/RefSeqCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/refseq/RefSeqCodec.java
index d94d9ff84..f142fa5aa 100644
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/refseq/RefSeqCodec.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/refseq/RefSeqCodec.java
@@ -12,19 +12,35 @@ import org.broadinstitute.sting.utils.exceptions.UserException;
 import java.util.ArrayList;
 
 /**
- * TODO FOR CHRIS HARTL
+ * Allows for reading in RefSeq information
  *
  * 

- * Codec Description
+ * Parses a sorted UCSC RefSeq file (see below) into relevant features: the gene name, the unique gene name (if multiple transcrips get separate entries), exons, gene start/stop, coding start/stop,
+ * strandedness of transcription. 
  * 

  *
  * 

- * See also: link to file specification
+ * Instructions for generating a RefSeq file for use with the RefSeq codec can be found on the Wiki here
+ * http://www.broadinstitute.org/gsa/wiki/index.php/RefSeq
  * 

+ * 
 Usage 

+ * The RefSeq Rod can be bound as any other rod, and is specified by REFSEQ, for example
+ * 
+ * -refSeqBinding:REFSEQ /path/to/refSeq.txt
+ * 

+ *
+ * You will need to consult individual walkers for the binding name ("refSeqBinding", above)
  *
  * 
File format example

+ * If you want to define your own file for use, the format is (tab delimited):
+ * bin, name, chrom, strand, transcription start, transcription end, coding start, coding end, num exons, exon starts, exon ends, id, alt. name, coding start status (complete/incomplete), coding end status (complete,incomplete)
+ * and exon frames, for example:
+ * 
+ * 76 NM_001011874 1 - 3204562 3661579 3206102 3661429 3 3204562,3411782,3660632, 3207049,3411982,3661579, 0 Xkr4 cmpl cmpl 1,2,0,
+ * 

+ * for more information see here
  * 

- *     A BAM file containing exactly one sample.
+ *     
  * 

  *
  * @author Mark DePristo
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodec.java
deleted file mode 100644
index 7f3d9e17d..000000000
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodec.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Copyright (c) 2011, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.utils.codecs.snpEff;
-
-import org.broad.tribble.Feature;
-import org.broad.tribble.FeatureCodec;
-import org.broad.tribble.TribbleException;
-import org.broad.tribble.readers.AsciiLineReader;
-import org.broad.tribble.readers.LineReader;
-import org.broadinstitute.sting.gatk.refdata.SelfScopingFeatureCodec;
-
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.EffectType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.ChangeType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.Zygosity;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-
-/**
- * Codec for decoding the output format of the SnpEff variant effect predictor tool
- *
- * 

- * This format has 23 tab-delimited fields:
- *
- * 
- * Chromosome
- * Position
- * Reference
- * Change
- * Change Type: {SNP, MNP, INS, DEL}
- * Zygosity: {Hom, Het}
- * Quality
- * Coverage
- * Warnings
- * Gene ID
- * Gene Name
- * Bio Type
- * Transcript ID
- * Exon ID
- * Exon Rank
- * Effect
- * Old/New Amino Acid
- * Old/New Codon
- * Codon Num
- * CDS Size
- * Codons Around
- * Amino Acids Around
- * Custom Interval ID
- * 

- * Note that we treat all except the Chromosome, Position, and Effect fields as optional.
- * 

- *
- * 

- * See also: @see SNPEff project page
- * 

- *
- * @author David Roazen
- * @since 2011
- */
-public class SnpEffCodec implements FeatureCodec, SelfScopingFeatureCodec {
-
-    public static final int EXPECTED_NUMBER_OF_FIELDS = 23;
-    public static final String FIELD_DELIMITER_PATTERN = "\\t";
-    public static final String EFFECT_FIELD_DELIMITER_PATTERN = "[,:]";
-    public static final String HEADER_LINE_START = "# ";
-    public static final String[] HEADER_FIELD_NAMES = { "Chromo",
-                                                        "Position",
-                                                        "Reference",
-                                                        "Change",
-                                                        "Change type",
-                                                        "Homozygous",
-                                                        "Quality",
-                                                        "Coverage",
-                                                        "Warnings",
-                                                        "Gene_ID",
-                                                        "Gene_name",
-                                                        "Bio_type",
-                                                        "Trancript_ID",   // yes, this is how it's spelled in the SnpEff output
-                                                        "Exon_ID",
-                                                        "Exon_Rank",
-                                                        "Effect",
-                                                        "old_AA/new_AA",
-                                                        "Old_codon/New_codon",
-                                                        "Codon_Num(CDS)",
-                                                        "CDS_size",
-                                                        "Codons around",
-                                                        "AAs around",
-                                                        "Custom_interval_ID"
-                                                      };
-
-    // The "Chromo", "Position", and "Effect" fields are required to be non-empty in every SnpEff output line:
-    public static final int[] REQUIRED_FIELDS = { 0, 1, 15 };
-
-    public static final String NON_CODING_GENE_FLAG = "WITHIN_NON_CODING_GENE";
-
-
-    public Feature decodeLoc ( String line ) {
-        return decode(line);
-    }
-
-    public Feature decode ( String line ) {
-        String[] tokens = line.split(FIELD_DELIMITER_PATTERN, -1);
-
-        if ( tokens.length != EXPECTED_NUMBER_OF_FIELDS ) {
-            throw new TribbleException.InvalidDecodeLine("Line does not have the expected (" + EXPECTED_NUMBER_OF_FIELDS +
-                                                         ") number of fields: found " + tokens.length + " fields.", line);
-        }
-
-        try {
-            trimAllFields(tokens);
-            checkForRequiredFields(tokens, line);
-
-            String contig = tokens[0];
-            long position = Long.parseLong(tokens[1]);
-
-            String reference = tokens[2].isEmpty() ? null : tokens[2];
-            String change = tokens[3].isEmpty() ? null : tokens[3];
-            ChangeType changeType = tokens[4].isEmpty() ? null : ChangeType.valueOf(tokens[4]);
-            Zygosity zygosity = tokens[5].isEmpty() ? null : Zygosity.valueOf(tokens[5]);
-            Double quality = tokens[6].isEmpty() ? null : Double.parseDouble(tokens[6]);
-            Long coverage = tokens[7].isEmpty() ? null : Long.parseLong(tokens[7]);
-            String warnings = tokens[8].isEmpty() ? null : tokens[8];
-            String geneID = tokens[9].isEmpty() ? null : tokens[9];
-            String geneName = tokens[10].isEmpty() ? null : tokens[10];
-            String bioType = tokens[11].isEmpty() ? null : tokens[11];
-            String transcriptID = tokens[12].isEmpty() ? null : tokens[12];
-            String exonID = tokens[13].isEmpty() ? null : tokens[13];
-            Integer exonRank = tokens[14].isEmpty() ? null : Integer.parseInt(tokens[14]);
-
-            boolean isNonCodingGene = isNonCodingGene(tokens[15]);
-
-            // Split the effect field into three subfields if the WITHIN_NON_CODING_GENE flag is present,
-            // otherwise split it into two subfields. We need this limit to prevent the extra effect-related information
-            // in the final field (when present) from being inappropriately tokenized:
-
-            int effectFieldTokenLimit = isNonCodingGene ? 3 : 2;
-            String[] effectFieldTokens = tokens[15].split(EFFECT_FIELD_DELIMITER_PATTERN, effectFieldTokenLimit);
-            EffectType effect = parseEffect(effectFieldTokens, isNonCodingGene);
-            String effectExtraInformation = parseEffectExtraInformation(effectFieldTokens, isNonCodingGene);
-
-            String oldAndNewAA = tokens[16].isEmpty() ? null : tokens[16];
-            String oldAndNewCodon = tokens[17].isEmpty() ? null : tokens[17];
-            Integer codonNum = tokens[18].isEmpty() ? null : Integer.parseInt(tokens[18]);
-            Integer cdsSize = tokens[19].isEmpty() ? null : Integer.parseInt(tokens[19]);
-            String codonsAround = tokens[20].isEmpty() ? null : tokens[20];
-            String aasAround = tokens[21].isEmpty() ? null : tokens[21];
-            String customIntervalID = tokens[22].isEmpty() ? null : tokens[22];
-
-            return new SnpEffFeature(contig, position, reference, change, changeType, zygosity, quality, coverage,
-                                     warnings, geneID, geneName, bioType, transcriptID, exonID, exonRank, isNonCodingGene,
-                                     effect, effectExtraInformation, oldAndNewAA, oldAndNewCodon, codonNum, cdsSize,
-                                     codonsAround, aasAround, customIntervalID);
-        }
-        catch ( NumberFormatException e ) {
-            throw new TribbleException.InvalidDecodeLine("Error parsing a numeric field : " + e.getMessage(), line);
-        }
-        catch ( IllegalArgumentException e ) {
-            throw new TribbleException.InvalidDecodeLine("Illegal value in field: " + e.getMessage(), line);
-        }
-    }
-
-    private void trimAllFields ( String[] tokens ) {
-        for ( int i = 0; i < tokens.length; i++ ) {
-            tokens[i] = tokens[i].trim();
-        }
-    }
-
-    private void checkForRequiredFields ( String[] tokens, String line ) {
-        for ( int requiredFieldIndex : REQUIRED_FIELDS ) {
-            if ( tokens[requiredFieldIndex].isEmpty() ) {
-                throw new TribbleException.InvalidDecodeLine("Line is missing required field \"" +
-                                                             HEADER_FIELD_NAMES[requiredFieldIndex] + "\"",
-                                                             line);
-            }
-        }
-    }
-
-    private boolean isNonCodingGene ( String effectField ) {
-        return effectField.startsWith(NON_CODING_GENE_FLAG);
-    }
-
-    private EffectType parseEffect ( String[] effectFieldTokens, boolean isNonCodingGene ) {
-        String effectName = "";
-
-        // If there's a WITHIN_NON_CODING_GENE flag, the effect name will be in the second subfield,
-        // otherwise it will be in the first subfield:
-
-        if ( effectFieldTokens.length > 1 && isNonCodingGene ) {
-            effectName = effectFieldTokens[1].trim();
-        }
-        else {
-            effectName = effectFieldTokens[0].trim();
-        }
-
-        return EffectType.valueOf(effectName);
-    }
-
-    private String parseEffectExtraInformation ( String[] effectFieldTokens, boolean isNonCodingGene ) {
-
-        // The extra effect-related information, if present, will always be the last subfield:
-
-        if ( (effectFieldTokens.length == 2 && ! isNonCodingGene) || effectFieldTokens.length == 3 ) {
-            return effectFieldTokens[effectFieldTokens.length - 1].trim();
-        }
-
-        return null;
-    }
-
-    public Class getFeatureType() {
-        return SnpEffFeature.class;
-    }
-
-    public Object readHeader ( LineReader reader ) {
-        String headerLine = "";
-
-        try {
-            headerLine = reader.readLine();
-        }
-        catch ( IOException e ) {
-            throw new TribbleException("Unable to read header line from input file.");
-        }
-
-        validateHeaderLine(headerLine);
-        return headerLine;
-    }
-
-    private void validateHeaderLine ( String headerLine ) {
-        if ( headerLine == null || ! headerLine.startsWith(HEADER_LINE_START) ) {
-            throw new TribbleException.InvalidHeader("Header line does not start with " + HEADER_LINE_START);
-        }
-
-        String[] headerTokens = headerLine.substring(HEADER_LINE_START.length()).split(FIELD_DELIMITER_PATTERN);
-
-        if ( headerTokens.length != EXPECTED_NUMBER_OF_FIELDS ) {
-            throw new TribbleException.InvalidHeader("Header line does not contain headings for the expected number (" +
-                                                     EXPECTED_NUMBER_OF_FIELDS + ") of columns.");
-        }
-
-        for ( int columnIndex = 0; columnIndex < headerTokens.length; columnIndex++ ) {
-            if ( ! HEADER_FIELD_NAMES[columnIndex].equals(headerTokens[columnIndex]) ) {
-                throw new TribbleException.InvalidHeader("Header field #" + columnIndex + ": Expected \"" +
-                                                         HEADER_FIELD_NAMES[columnIndex] + "\" but found \"" +
-                                                         headerTokens[columnIndex] + "\"");
-            }
-        }
-    }
-
-    public boolean canDecode ( final File potentialInput ) {
-        try {
-            LineReader reader = new AsciiLineReader(new FileInputStream(potentialInput));
-            readHeader(reader);
-        }
-        catch ( Exception e ) {
-            return false;
-        }
-
-        return true;
-    }
-}
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffConstants.java b/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffConstants.java
deleted file mode 100644
index 270db470f..000000000
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffConstants.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright (c) 2011, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.utils.codecs.snpEff;
-
-/**
- * A set of constants associated with the SnpEff codec.
- *
- * @author David Roazen
- */
-public class SnpEffConstants {
-
-    // Possible SnpEff biological effects and their associated impacts:
-    public enum EffectType {
-        START_GAINED                      (EffectImpact.HIGH),
-        START_LOST                        (EffectImpact.HIGH),
-        EXON_DELETED                      (EffectImpact.HIGH),
-        FRAME_SHIFT                       (EffectImpact.HIGH),
-        STOP_GAINED                       (EffectImpact.HIGH),
-        STOP_LOST                         (EffectImpact.HIGH),
-        SPLICE_SITE_ACCEPTOR              (EffectImpact.HIGH),
-        SPLICE_SITE_DONOR                 (EffectImpact.HIGH),
-
-        NON_SYNONYMOUS_CODING             (EffectImpact.MODERATE),
-        UTR_5_DELETED                     (EffectImpact.MODERATE),
-        UTR_3_DELETED                     (EffectImpact.MODERATE),
-        CODON_INSERTION                   (EffectImpact.MODERATE),
-        CODON_CHANGE_PLUS_CODON_INSERTION (EffectImpact.MODERATE),
-        CODON_DELETION                    (EffectImpact.MODERATE),
-        CODON_CHANGE_PLUS_CODON_DELETION  (EffectImpact.MODERATE),
-
-        NONE                              (EffectImpact.LOW),
-        CHROMOSOME                        (EffectImpact.LOW),
-        INTERGENIC                        (EffectImpact.LOW),
-        UPSTREAM                          (EffectImpact.LOW),
-        UTR_5_PRIME                       (EffectImpact.LOW),
-        SYNONYMOUS_START                  (EffectImpact.LOW),
-        NON_SYNONYMOUS_START              (EffectImpact.LOW),
-        CDS                               (EffectImpact.LOW),
-        GENE                              (EffectImpact.LOW),
-        TRANSCRIPT                        (EffectImpact.LOW),
-        EXON                              (EffectImpact.LOW),
-        SYNONYMOUS_CODING                 (EffectImpact.LOW),
-        CODON_CHANGE                      (EffectImpact.LOW),
-        SYNONYMOUS_STOP                   (EffectImpact.LOW),
-        NON_SYNONYMOUS_STOP               (EffectImpact.LOW),
-        INTRON                            (EffectImpact.LOW),
-        UTR_3_PRIME                       (EffectImpact.LOW),
-        DOWNSTREAM                        (EffectImpact.LOW),
-        INTRON_CONSERVED                  (EffectImpact.LOW),
-        INTERGENIC_CONSERVED              (EffectImpact.LOW),
-        CUSTOM                            (EffectImpact.LOW);
-
-        private final EffectImpact impact;
-
-        EffectType ( EffectImpact impact ) {
-            this.impact = impact;
-        }
-
-        public EffectImpact getImpact() {
-            return impact;
-        }
-    }
-
-    public enum EffectImpact {
-        LOW       (1),
-        MODERATE  (2),
-        HIGH      (3);
-
-        private final int severityRating;
-
-        EffectImpact ( int severityRating ) {
-            this.severityRating = severityRating;
-        }
-
-        public boolean isHigherImpactThan ( EffectImpact other ) {
-            return this.severityRating > other.severityRating;
-        }
-    }
-
-    // The kinds of variants supported by the SnpEff output format:
-    public enum ChangeType {
-        SNP,
-        MNP,
-        INS,
-        DEL
-    }
-
-    // Possible zygosities of SnpEff variants:
-    public enum Zygosity {
-        Hom,
-        Het
-    }
-}
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffFeature.java b/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffFeature.java
deleted file mode 100644
index 2f120b7d2..000000000
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffFeature.java
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Copyright (c) 2011, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.utils.codecs.snpEff;
-
-import org.broad.tribble.Feature;
-
-import java.util.NoSuchElementException;
-
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.EffectType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.EffectImpact;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.ChangeType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.Zygosity;
-
-/**
- * Feature returned by the SnpEff codec -- stores the parsed field values from a line of SnpEff output.
- *
- * Many fields are optional, and missing values are represented by nulls. You should always call the
- * hasX() method before calling the corresponding getX() method. Required fields can never be null
- * and do not have a hasX() method.
- *
- * @author David Roazen
- */
-public class SnpEffFeature implements Feature {
-
-    private String contig;                   // REQUIRED FIELD
-    private long position;                   // REQUIRED FIELD
-    private String reference;
-    private String change;
-    private ChangeType changeType;
-    private Zygosity zygosity;
-    private Double quality;
-    private Long coverage;
-    private String warnings;
-    private String geneID;
-    private String geneName;
-    private String bioType;
-    private String transcriptID;
-    private String exonID;
-    private Integer exonRank;
-    private boolean isNonCodingGene;         // REQUIRED FIELD
-    private EffectType effect;               // REQUIRED FIELD
-    private String effectExtraInformation;
-    private String oldAndNewAA;
-    private String oldAndNewCodon;
-    private Integer codonNum;
-    private Integer cdsSize;
-    private String codonsAround;
-    private String aasAround;
-    private String customIntervalID;
-
-    public SnpEffFeature ( String contig,
-                           long position,
-                           String reference,
-                           String change,
-                           ChangeType changeType,
-                           Zygosity zygosity,
-                           Double quality,
-                           Long coverage,
-                           String warnings,
-                           String geneID,
-                           String geneName,
-                           String bioType,
-                           String transcriptID,
-                           String exonID,
-                           Integer exonRank,
-                           boolean isNonCodingGene,
-                           EffectType effect,
-                           String effectExtraInformation,
-                           String oldAndNewAA,
-                           String oldAndNewCodon,
-                           Integer codonNum,
-                           Integer cdsSize,
-                           String codonsAround,
-                           String aasAround,
-                           String customIntervalID ) {
-
-        if ( contig == null || effect == null ) {
-            throw new IllegalArgumentException("contig and effect cannot be null, as they are required fields");
-        }
-
-        this.contig = contig;
-        this.position = position;
-        this.reference = reference;
-        this.change = change;
-        this.changeType = changeType;
-        this.zygosity = zygosity;
-        this.quality = quality;
-        this.coverage = coverage;
-        this.warnings = warnings;
-        this.geneID = geneID;
-        this.geneName = geneName;
-        this.bioType = bioType;
-        this.transcriptID = transcriptID;
-        this.exonID = exonID;
-        this.exonRank = exonRank;
-        this.isNonCodingGene = isNonCodingGene;
-        this.effect = effect;
-        this.effectExtraInformation = effectExtraInformation;
-        this.oldAndNewAA = oldAndNewAA;
-        this.oldAndNewCodon = oldAndNewCodon;
-        this.codonNum = codonNum;
-        this.cdsSize = cdsSize;
-        this.codonsAround = codonsAround;
-        this.aasAround = aasAround;
-        this.customIntervalID = customIntervalID;
-    }
-
-    public boolean isHigherImpactThan ( SnpEffFeature other ) {
-
-        // If one effect is in a non-coding gene and the other is not, the effect NOT in the
-        // non-coding gene has higher impact:
-
-        if ( ! isNonCodingGene() && other.isNonCodingGene() ) {
-            return true;
-        }
-        else if ( isNonCodingGene() && ! other.isNonCodingGene() ) {
-            return false;
-        }
-
-        // Otherwise, both effects are either in or not in a non-coding gene, so we compare the impacts
-        // of the effects themselves as defined in the SnpEffConstants class:
-
-        return getEffectImpact().isHigherImpactThan(other.getEffectImpact());
-    }
-
-    public String getChr() {
-        return contig;
-    }
-
-    public int getStart() {
-        return (int)position;
-    }
-
-    public int getEnd() {
-        return (int)position;
-    }
-
-    public boolean hasReference() {
-        return reference != null;
-    }
-
-    public String getReference() {
-        if ( reference == null ) throw new NoSuchElementException("This feature has no reference field");
-        return reference;
-    }
-
-    public boolean hasChange() {
-        return change != null;
-    }
-
-    public String getChange() {
-        if ( change == null ) throw new NoSuchElementException("This feature has no change field");
-        return change;
-    }
-
-    public boolean hasChangeType() {
-        return changeType != null;
-    }
-
-    public ChangeType getChangeType() {
-        if ( changeType == null ) throw new NoSuchElementException("This feature has no changeType field");
-        return changeType;
-    }
-
-    public boolean hasZygosity() {
-        return zygosity != null;
-    }
-
-    public Zygosity getZygosity() {
-        if ( zygosity == null ) throw new NoSuchElementException("This feature has no zygosity field");
-        return zygosity;
-    }
-
-    public boolean hasQuality() {
-        return quality != null;
-    }
-
-    public Double getQuality() {
-        if ( quality == null ) throw new NoSuchElementException("This feature has no quality field");
-        return quality;
-    }
-
-    public boolean hasCoverage() {
-        return coverage != null;
-    }
-
-    public Long getCoverage() {
-        if ( coverage == null ) throw new NoSuchElementException("This feature has no coverage field");
-        return coverage;
-    }
-
-    public boolean hasWarnings() {
-        return warnings != null;
-    }
-
-    public String getWarnings() {
-        if ( warnings == null ) throw new NoSuchElementException("This feature has no warnings field");
-        return warnings;
-    }
-
-    public boolean hasGeneID() {
-        return geneID != null;
-    }
-
-    public String getGeneID() {
-        if ( geneID == null ) throw new NoSuchElementException("This feature has no geneID field");
-        return geneID;
-    }
-
-    public boolean hasGeneName() {
-        return geneName != null;
-    }
-
-    public String getGeneName() {
-        if ( geneName == null ) throw new NoSuchElementException("This feature has no geneName field");
-        return geneName;
-    }
-
-    public boolean hasBioType() {
-        return bioType != null;
-    }
-
-    public String getBioType() {
-        if ( bioType == null ) throw new NoSuchElementException("This feature has no bioType field");
-        return bioType;
-    }
-
-    public boolean hasTranscriptID() {
-        return transcriptID != null;
-    }
-
-    public String getTranscriptID() {
-        if ( transcriptID == null ) throw new NoSuchElementException("This feature has no transcriptID field");
-        return transcriptID;
-    }
-
-    public boolean hasExonID() {
-        return exonID != null;
-    }
-
-    public String getExonID() {
-        if ( exonID == null ) throw new NoSuchElementException("This feature has no exonID field");
-        return exonID;
-    }
-
-    public boolean hasExonRank() {
-        return exonRank != null;
-    }
-
-    public Integer getExonRank() {
-        if ( exonRank == null ) throw new NoSuchElementException("This feature has no exonRank field");
-        return exonRank;
-    }
-
-    public boolean isNonCodingGene() {
-        return isNonCodingGene;
-    }
-
-    public EffectType getEffect() {
-        return effect;
-    }
-
-    public EffectImpact getEffectImpact() {
-        return effect.getImpact();
-    }
-
-    public boolean hasEffectExtraInformation() {
-        return effectExtraInformation != null;
-    }
-
-    public String getEffectExtraInformation() {
-        if ( effectExtraInformation == null ) throw new NoSuchElementException("This feature has no effectExtraInformation field");
-        return effectExtraInformation;
-    }
-
-    public boolean hasOldAndNewAA() {
-        return oldAndNewAA != null;
-    }
-
-    public String getOldAndNewAA() {
-        if ( oldAndNewAA == null ) throw new NoSuchElementException("This feature has no oldAndNewAA field");
-        return oldAndNewAA;
-    }
-
-    public boolean hasOldAndNewCodon() {
-        return oldAndNewCodon != null;
-    }
-
-    public String getOldAndNewCodon() {
-        if ( oldAndNewCodon == null ) throw new NoSuchElementException("This feature has no oldAndNewCodon field");
-        return oldAndNewCodon;
-    }
-
-    public boolean hasCodonNum() {
-        return codonNum != null;
-    }
-
-    public Integer getCodonNum() {
-        if ( codonNum == null ) throw new NoSuchElementException("This feature has no codonNum field");
-        return codonNum;
-    }
-
-    public boolean hasCdsSize() {
-        return cdsSize != null;
-    }
-
-    public Integer getCdsSize() {
-        if ( cdsSize == null ) throw new NoSuchElementException("This feature has no cdsSize field");
-        return cdsSize;
-    }
-
-    public boolean hasCodonsAround() {
-        return codonsAround != null;
-    }
-
-    public String getCodonsAround() {
-        if ( codonsAround == null ) throw new NoSuchElementException("This feature has no codonsAround field");
-        return codonsAround;
-    }
-
-    public boolean hadAasAround() {
-        return aasAround != null;
-    }
-
-    public String getAasAround() {
-        if ( aasAround == null ) throw new NoSuchElementException("This feature has no aasAround field");
-        return aasAround;
-    }
-
-    public boolean hasCustomIntervalID() {
-        return customIntervalID != null;
-    }
-
-    public String getCustomIntervalID() {
-        if ( customIntervalID == null ) throw new NoSuchElementException("This feature has no customIntervalID field");
-        return customIntervalID;
-    }
-
-    public boolean equals ( Object o ) {
-        if ( o == null || ! (o instanceof SnpEffFeature) ) {
-            return false;
-        }
-
-        SnpEffFeature other = (SnpEffFeature)o;
-
-        return contig.equals(other.contig) &&
-               position == other.position &&
-               (reference == null ? other.reference == null : reference.equals(other.reference)) &&
-               (change == null ? other.change == null : change.equals(other.change)) &&
-               changeType == other.changeType &&
-               zygosity == other.zygosity &&
-               (quality == null ? other.quality == null : quality.equals(other.quality)) &&
-               (coverage == null ? other.coverage == null : coverage.equals(other.coverage)) &&
-               (warnings == null ? other.warnings == null : warnings.equals(other.warnings)) &&
-               (geneID == null ? other.geneID == null : geneID.equals(other.geneID)) &&
-               (geneName == null ? other.geneName == null : geneName.equals(other.geneName)) &&
-               (bioType == null ? other.bioType == null : bioType.equals(other.bioType)) &&
-               (transcriptID == null ? other.transcriptID == null : transcriptID.equals(other.transcriptID)) &&
-               (exonID == null ? other.exonID == null : exonID.equals(other.exonID)) &&
-               (exonRank == null ? other.exonRank == null : exonRank.equals(other.exonRank)) &&
-               isNonCodingGene == other.isNonCodingGene &&
-               effect == other.effect &&
-               (effectExtraInformation == null ? other.effectExtraInformation == null : effectExtraInformation.equals(other.effectExtraInformation)) &&
-               (oldAndNewAA == null ? other.oldAndNewAA == null : oldAndNewAA.equals(other.oldAndNewAA)) &&
-               (oldAndNewCodon == null ? other.oldAndNewCodon == null : oldAndNewCodon.equals(other.oldAndNewCodon)) &&
-               (codonNum == null ? other.codonNum == null : codonNum.equals(other.codonNum)) &&
-               (cdsSize == null ? other.cdsSize == null : cdsSize.equals(other.cdsSize)) &&
-               (codonsAround == null ? other.codonsAround == null : codonsAround.equals(other.codonsAround)) &&
-               (aasAround == null ? other.aasAround == null : aasAround.equals(other.aasAround)) &&
-               (customIntervalID == null ? other.customIntervalID == null : customIntervalID.equals(other.customIntervalID));
-    }
-
-    public String toString() {
-        return "[Contig: " + contig +
-               " Position: " + position +
-               " Reference: " + reference +
-               " Change: " + change +
-               " Change Type: " + changeType +
-               " Zygosity: " + zygosity +
-               " Quality: " + quality +
-               " Coverage: " + coverage +
-               " Warnings: " + warnings +
-               " Gene ID: " + geneID +
-               " Gene Name: " + geneName +
-               " Bio Type: " + bioType +
-               " Transcript ID: " + transcriptID +
-               " Exon ID: " + exonID +
-               " Exon Rank: " + exonRank +
-               " Non-Coding Gene: " + isNonCodingGene +
-               " Effect: " + effect +
-               " Effect Extra Information: " + effectExtraInformation +
-               " Old/New AA: " + oldAndNewAA +
-               " Old/New Codon: " + oldAndNewCodon +
-               " Codon Num: " + codonNum +
-               " CDS Size: " + cdsSize +
-               " Codons Around: " + codonsAround +
-               " AAs Around: " + aasAround +
-               " Custom Interval ID: " + customIntervalID +
-               "]";
-    }
-}
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java
index bb212e128..43b07476d 100755
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/AbstractVCFCodec.java
@@ -6,6 +6,7 @@ import org.broad.tribble.FeatureCodec;
 import org.broad.tribble.NameAwareCodec;
 import org.broad.tribble.TribbleException;
 import org.broad.tribble.readers.LineReader;
+import org.broad.tribble.util.BlockCompressedInputStream;
 import org.broad.tribble.util.ParsingUtils;
 import org.broadinstitute.sting.gatk.refdata.SelfScopingFeatureCodec;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
@@ -114,15 +115,21 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec,
                     }
                     arrayIndex++;
                 }
+
+                boolean sawFormatTag = false;
                 if ( arrayIndex < strings.length ) {
                     if ( !strings[arrayIndex].equals("FORMAT") )
                         throw new TribbleException.InvalidHeader("we were expecting column name 'FORMAT' but we saw '" + strings[arrayIndex] + "'");
+                    sawFormatTag = true;
                     arrayIndex++;
                 }
 
-                while (arrayIndex < strings.length)
+                while ( arrayIndex < strings.length )
                     auxTags.add(strings[arrayIndex++]);
 
+                if ( sawFormatTag && auxTags.size() == 0 )
+                    throw new UserException.MalformedVCFHeader("The FORMAT field was provided but there is no genotype/sample data");
+
             } else {
                 if ( str.startsWith("##INFO=") ) {
                     VCFInfoHeaderLine info = new VCFInfoHeaderLine(str.substring(7),version);
@@ -199,35 +206,31 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec,
      * @return a VariantContext
      */
     public Feature decode(String line) {
-        return reallyDecode(line);
-    }
+        // the same line reader is not used for parsing the header and parsing lines, if we see a #, we've seen a header line
+        if (line.startsWith(VCFHeader.HEADER_INDICATOR)) return null;
 
-    private Feature reallyDecode(String line) {
-            // the same line reader is not used for parsing the header and parsing lines, if we see a #, we've seen a header line
-            if (line.startsWith(VCFHeader.HEADER_INDICATOR)) return null;
+        // our header cannot be null, we need the genotype sample names and counts
+        if (header == null) throw new ReviewedStingException("VCF Header cannot be null when decoding a record");
 
-            // our header cannot be null, we need the genotype sample names and counts
-            if (header == null) throw new ReviewedStingException("VCF Header cannot be null when decoding a record");
+        if (parts == null)
+            parts = new String[Math.min(header.getColumnCount(), NUM_STANDARD_FIELDS+1)];
 
-            if (parts == null)
-                parts = new String[Math.min(header.getColumnCount(), NUM_STANDARD_FIELDS+1)];
+        int nParts = ParsingUtils.split(line, parts, VCFConstants.FIELD_SEPARATOR_CHAR, true);
 
-            int nParts = ParsingUtils.split(line, parts, VCFConstants.FIELD_SEPARATOR_CHAR, true);
+        // if we have don't have a header, or we have a header with no genotyping data check that we have eight columns.  Otherwise check that we have nine (normal colummns + genotyping data)
+        if (( (header == null || !header.hasGenotypingData()) && nParts != NUM_STANDARD_FIELDS) ||
+             (header != null && header.hasGenotypingData() && nParts != (NUM_STANDARD_FIELDS + 1)) )
+            throw new UserException.MalformedVCF("there aren't enough columns for line " + line + " (we expected " + (header == null ? NUM_STANDARD_FIELDS : NUM_STANDARD_FIELDS + 1) +
+                    " tokens, and saw " + nParts + " )", lineNo);
 
-            // if we have don't have a header, or we have a header with no genotyping data check that we have eight columns.  Otherwise check that we have nine (normal colummns + genotyping data)
-            if (( (header == null || (header != null && !header.hasGenotypingData())) && nParts != NUM_STANDARD_FIELDS) ||
-                 (header != null && header.hasGenotypingData() && nParts != (NUM_STANDARD_FIELDS + 1)) )
-                throw new UserException.MalformedVCF("there aren't enough columns for line " + line + " (we expected " + (header == null ? NUM_STANDARD_FIELDS : NUM_STANDARD_FIELDS + 1) +
-                        " tokens, and saw " + nParts + " )", lineNo);
-
-            return parseVCFLine(parts);
+        return parseVCFLine(parts);
     }
 
     protected void generateException(String message) {
         throw new UserException.MalformedVCF(message, lineNo);
     }
 
-    private static void generateException(String message, int lineNo) {
+    protected static void generateException(String message, int lineNo) {
         throw new UserException.MalformedVCF(message, lineNo);
     }
 
@@ -345,6 +348,9 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec,
             generateException("The VCF specification requires a valid info field");
 
         if ( !infoField.equals(VCFConstants.EMPTY_INFO_FIELD) ) {
+            if ( infoField.indexOf("\t") != -1 || infoField.indexOf(" ") != -1 )
+                generateException("The VCF specification does not allow for whitespace in the INFO field");
+
             int infoValueSplitSize = ParsingUtils.split(infoField, infoValueArray, VCFConstants.INFO_FIELD_SEPARATOR_CHAR);
             for (int i = 0; i < infoValueSplitSize; i++) {
                 String key;
@@ -587,7 +593,8 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec,
     public final static boolean canDecodeFile(final File potentialInput, final String MAGIC_HEADER_LINE) {
         try {
             return isVCFStream(new FileInputStream(potentialInput), MAGIC_HEADER_LINE) ||
-                    isVCFStream(new GZIPInputStream(new FileInputStream(potentialInput)), MAGIC_HEADER_LINE);
+                    isVCFStream(new GZIPInputStream(new FileInputStream(potentialInput)), MAGIC_HEADER_LINE) ||
+                    isVCFStream(new BlockCompressedInputStream(new FileInputStream(potentialInput)), MAGIC_HEADER_LINE);
         } catch ( FileNotFoundException e ) {
             return false;
         } catch ( IOException e ) {
@@ -598,12 +605,17 @@ public abstract class AbstractVCFCodec implements FeatureCodec, NameAwareCodec,
     private final static boolean isVCFStream(final InputStream stream, final String MAGIC_HEADER_LINE) {
         try {
             byte[] buff = new byte[MAGIC_HEADER_LINE.length()];
-            stream.read(buff, 0, MAGIC_HEADER_LINE.length());
-            String firstLine = new String(buff);
-            stream.close();
-            return firstLine.startsWith(MAGIC_HEADER_LINE);
+            int nread = stream.read(buff, 0, MAGIC_HEADER_LINE.length());
+            boolean eq = Arrays.equals(buff, MAGIC_HEADER_LINE.getBytes());
+            return eq;
+//            String firstLine = new String(buff);
+//            return firstLine.startsWith(MAGIC_HEADER_LINE);
         } catch ( IOException e ) {
             return false;
+        } catch ( RuntimeException e ) {
+            return false;
+        } finally {
+            try { stream.close(); } catch ( IOException e ) {}
         }
     }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java
new file mode 100644
index 000000000..71ec4ce1b
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/IndexingVCFWriter.java
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.codecs.vcf;
+
+import com.google.java.contract.Ensures;
+import com.google.java.contract.Requires;
+import net.sf.samtools.SAMSequenceDictionary;
+import org.broad.tribble.Tribble;
+import org.broad.tribble.TribbleException;
+import org.broad.tribble.index.DynamicIndexCreator;
+import org.broad.tribble.index.Index;
+import org.broad.tribble.index.IndexFactory;
+import org.broad.tribble.util.LittleEndianOutputStream;
+import org.broad.tribble.util.PositionalStream;
+import org.broadinstitute.sting.gatk.refdata.tracks.IndexDictionaryUtils;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.io.*;
+
+/**
+ * this class writes VCF files
+ */
+public abstract class IndexingVCFWriter implements VCFWriter {
+    final private String name;
+    private final SAMSequenceDictionary refDict;
+
+    private OutputStream outputStream;
+    private PositionalStream positionalStream = null;
+    private DynamicIndexCreator indexer = null;
+    private LittleEndianOutputStream idxStream = null;
+
+    @Requires({"name != null",
+            "! ( location == null && output == null )",
+            "! ( enableOnTheFlyIndexing && location == null )"})
+    protected IndexingVCFWriter(final String name, final File location, final OutputStream output, final SAMSequenceDictionary refDict, final boolean enableOnTheFlyIndexing) {
+        outputStream = output;
+        this.name = name;
+        this.refDict = refDict;
+
+        if ( enableOnTheFlyIndexing ) {
+            try {
+                idxStream = new LittleEndianOutputStream(new FileOutputStream(Tribble.indexFile(location)));
+                //System.out.println("Creating index on the fly for " + location);
+                indexer = new DynamicIndexCreator(IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME);
+                indexer.initialize(location, indexer.defaultBinSize());
+                positionalStream = new PositionalStream(output);
+                outputStream = positionalStream;
+            } catch ( IOException ex ) {
+                // No matter what we keep going, since we don't care if we can't create the index file
+                idxStream = null;
+                indexer = null;
+                positionalStream = null;
+            }
+        }
+    }
+
+    @Ensures("result != null")
+    public OutputStream getOutputStream() {
+        return outputStream;
+    }
+
+    @Ensures("result != null")
+    public String getStreamName() {
+        return name;
+    }
+
+    public abstract void writeHeader(VCFHeader header);
+
+    /**
+     * attempt to close the VCF file
+     */
+    public void close() {
+        // try to close the index stream (keep it separate to help debugging efforts)
+        if ( indexer != null ) {
+            try {
+                Index index = indexer.finalizeIndex(positionalStream.getPosition());
+                IndexDictionaryUtils.setIndexSequenceDictionary(index, refDict);
+                index.write(idxStream);
+                idxStream.close();
+            } catch (IOException e) {
+                throw new ReviewedStingException("Unable to close index for " + getStreamName(), e);
+            }
+        }
+    }
+
+    /**
+     * add a record to the file
+     *
+     * @param vc      the Variant Context object
+     */
+    public void add(VariantContext vc) {
+        // if we are doing on the fly indexing, add the record ***before*** we write any bytes
+        if ( indexer != null )
+            indexer.addFeature(vc, positionalStream.getPosition());
+    }
+
+    /**
+     * Returns a reasonable "name" for this writer, to display to the user if something goes wrong
+     *
+     * @param location
+     * @param stream
+     * @return
+     */
+    protected static final String writerName(final File location, final OutputStream stream) {
+        return location == null ? stream.toString() : location.getAbsolutePath();
+    }
+
+    /**
+     * Returns a output stream writing to location, or throws a UserException if this fails
+     * @param location
+     * @return
+     */
+    protected static OutputStream openOutputStream(final File location) {
+        try {
+            return new FileOutputStream(location);
+        } catch (FileNotFoundException e) {
+            throw new UserException.CouldNotCreateOutputFile(location, "Unable to create VCF writer", e);
+        }
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/StandardVCFWriter.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/StandardVCFWriter.java
index d3705813c..0da7a100f 100755
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/StandardVCFWriter.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/StandardVCFWriter.java
@@ -24,6 +24,7 @@
 
 package org.broadinstitute.sting.utils.codecs.vcf;
 
+import net.sf.samtools.SAMSequenceDictionary;
 import org.broad.tribble.Tribble;
 import org.broad.tribble.TribbleException;
 import org.broad.tribble.index.DynamicIndexCreator;
@@ -44,46 +45,30 @@ import java.util.*;
 /**
  * this class writes VCF files
  */
-public class StandardVCFWriter implements VCFWriter {
+public class StandardVCFWriter extends IndexingVCFWriter {
+    // the print stream we're writing to
+    final protected BufferedWriter mWriter;
+
+    // should we write genotypes or just sites?
+    final protected boolean doNotWriteGenotypes;
 
     // the VCF header we're storing
     protected VCFHeader mHeader = null;
 
-    // the print stream we're writing to
-    protected BufferedWriter mWriter;
-    protected PositionalStream positionalStream = null;
-
     // were filters applied?
     protected boolean filtersWereAppliedToContext = false;
 
-    // should we write genotypes or just sites?
-    protected boolean doNotWriteGenotypes = false;
-
-    protected DynamicIndexCreator indexer = null;
-    protected File indexFile = null;
-    LittleEndianOutputStream idxStream = null;
-    File location = null;
-
     /**         
      * create a VCF writer, given a file to write to
      *
      * @param location the file location to write to
      */
-    public StandardVCFWriter(File location) {
-        this(location, openOutputStream(location), true, false);
+    public StandardVCFWriter(final File location, final SAMSequenceDictionary refDict) {
+        this(location, openOutputStream(location), refDict, true, false);
     }
 
-    public StandardVCFWriter(File location, boolean enableOnTheFlyIndexing) {
-        this(location, openOutputStream(location), enableOnTheFlyIndexing, false);
-    }
-
-    /**
-     * create a VCF writer, given a stream to write to
-     *
-     * @param output   the file location to write to
-     */
-    public StandardVCFWriter(OutputStream output) {
-        this(output, false);
+    public StandardVCFWriter(File location, final SAMSequenceDictionary refDict, boolean enableOnTheFlyIndexing) {
+        this(location, openOutputStream(location), refDict, enableOnTheFlyIndexing, false);
     }
 
     /**
@@ -92,33 +77,23 @@ public class StandardVCFWriter implements VCFWriter {
      * @param output   the file location to write to
      * @param doNotWriteGenotypes   do not write genotypes
      */
-    public StandardVCFWriter(OutputStream output, boolean doNotWriteGenotypes) {
-        mWriter = new BufferedWriter(new OutputStreamWriter(output));
+    public StandardVCFWriter(final OutputStream output, final SAMSequenceDictionary refDict, final boolean doNotWriteGenotypes) {
+        this(null, output, refDict, false, doNotWriteGenotypes);
+    }
+
+    public StandardVCFWriter(final File location, final OutputStream output, final SAMSequenceDictionary refDict, final boolean enableOnTheFlyIndexing, boolean doNotWriteGenotypes) {
+        super(writerName(location, output), location, output, refDict, enableOnTheFlyIndexing);
+        mWriter = new BufferedWriter(new OutputStreamWriter(getOutputStream())); // todo -- fix buffer size
         this.doNotWriteGenotypes = doNotWriteGenotypes;
     }
 
-    public StandardVCFWriter(File location, OutputStream output, boolean enableOnTheFlyIndexing, boolean doNotWriteGenotypes) {
-        this.location = location;
-
-        if ( enableOnTheFlyIndexing ) {
-            indexFile = Tribble.indexFile(location);
-            try {
-                idxStream = new LittleEndianOutputStream(new FileOutputStream(indexFile));
-                //System.out.println("Creating index on the fly for " + location);
-                indexer = new DynamicIndexCreator(IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME);
-                indexer.initialize(location, indexer.defaultBinSize());
-                positionalStream = new PositionalStream(output);
-                output = positionalStream;
-            } catch ( IOException ex ) {
-                // No matter what we keep going, since we don't care if we can't create the index file
-            }
-        }
-
-        //mWriter = new BufferedWriter(new OutputStreamWriter(new PositionalStream(output)));
-        mWriter = new BufferedWriter(new OutputStreamWriter(output));
-        this.doNotWriteGenotypes = doNotWriteGenotypes;
-    }
+    // --------------------------------------------------------------------------------
+    //
+    // VCFWriter interface functions
+    //
+    // --------------------------------------------------------------------------------
 
+    @Override
     public void writeHeader(VCFHeader header) {
         mHeader = doNotWriteGenotypes ? new VCFHeader(header.getMetaData()) : header;
         
@@ -158,44 +133,24 @@ public class StandardVCFWriter implements VCFWriter {
             mWriter.flush();  // necessary so that writing to an output stream will work
         }
         catch (IOException e) {
-            throw new TribbleException("IOException writing the VCF header to " + locationString(), e);
+            throw new ReviewedStingException("IOException writing the VCF header to " + getStreamName(), e);
         }
     }
 
-    private String locationString() {
-        return location == null ? mWriter.toString() : location.getAbsolutePath();
-    }
-
     /**
      * attempt to close the VCF file
      */
+    @Override
     public void close() {
         // try to close the vcf stream
         try {
             mWriter.flush();
             mWriter.close();
         } catch (IOException e) {
-            throw new TribbleException("Unable to close " + locationString() + " because of " + e.getMessage());
+            throw new ReviewedStingException("Unable to close " + getStreamName(), e);
         }
 
-        // try to close the index stream (keep it separate to help debugging efforts)
-        if ( indexer != null ) {
-            try {
-                Index index = indexer.finalizeIndex(positionalStream.getPosition());
-                index.write(idxStream);
-                idxStream.close();
-            } catch (IOException e) {
-                throw new TribbleException("Unable to close index for " + locationString() + " because of " + e.getMessage());
-            }
-        }
-    }
-
-    protected static OutputStream openOutputStream(File location) {
-        try {
-            return new FileOutputStream(location);
-        } catch (FileNotFoundException e) {
-            throw new TribbleException("Unable to create VCF file at location: " + location);
-        }
+        super.close();
     }
 
     /**
@@ -203,28 +158,17 @@ public class StandardVCFWriter implements VCFWriter {
      *
      * @param vc      the Variant Context object
      */
+    @Override
     public void add(VariantContext vc) {
-        add(vc, false);
-    }
-
-    /**
-     * add a record to the file
-     *
-     * @param vc      the Variant Context object
-     * @param refBaseShouldBeAppliedToEndOfAlleles *** THIS SHOULD BE FALSE EXCEPT FOR AN INDEL AT THE EXTREME BEGINNING OF A CONTIG (WHERE THERE IS NO PREVIOUS BASE, SO WE USE THE BASE AFTER THE EVENT INSTEAD)
-     */
-    public void add(VariantContext vc, boolean refBaseShouldBeAppliedToEndOfAlleles) {
         if ( mHeader == null )
-            throw new IllegalStateException("The VCF Header must be written before records can be added: " + locationString());
+            throw new IllegalStateException("The VCF Header must be written before records can be added: " + getStreamName());
 
         if ( doNotWriteGenotypes )
             vc = VariantContext.modifyGenotypes(vc, null);
 
         try {
-            vc = VariantContext.createVariantContextWithPaddedAlleles(vc, refBaseShouldBeAppliedToEndOfAlleles);
-
-            // if we are doing on the fly indexing, add the record ***before*** we write any bytes 
-            if ( indexer != null ) indexer.addFeature(vc, positionalStream.getPosition());
+            vc = VariantContext.createVariantContextWithPaddedAlleles(vc, false);
+            super.add(vc);
 
             Map alleleMap = new HashMap(vc.getAlleles().size());
             alleleMap.put(Allele.NO_CALL, VCFConstants.EMPTY_ALLELE); // convenience for lookup
@@ -275,7 +219,7 @@ public class StandardVCFWriter implements VCFWriter {
             mWriter.write(VCFConstants.FIELD_SEPARATOR);
 
             // FILTER
-            String filters = vc.isFiltered() ? ParsingUtils.join(";", ParsingUtils.sortList(vc.getFilters())) : (filtersWereAppliedToContext || vc.filtersWereApplied() ? VCFConstants.PASSES_FILTERS_v4 : VCFConstants.UNFILTERED);
+            String filters = getFilterString(vc, filtersWereAppliedToContext);
             mWriter.write(filters);
             mWriter.write(VCFConstants.FIELD_SEPARATOR);
 
@@ -317,9 +261,22 @@ public class StandardVCFWriter implements VCFWriter {
             mWriter.write("\n");
             mWriter.flush();  // necessary so that writing to an output stream will work
         } catch (IOException e) {
-            throw new RuntimeException("Unable to write the VCF object to " + locationString());
+            throw new RuntimeException("Unable to write the VCF object to " + getStreamName());
         }
+    }
 
+    // --------------------------------------------------------------------------------
+    //
+    // implementation functions
+    //
+    // --------------------------------------------------------------------------------
+
+    public static final String getFilterString(final VariantContext vc) {
+        return getFilterString(vc, false);
+    }
+
+    public static final String getFilterString(final VariantContext vc, boolean forcePASS) {
+        return vc.isFiltered() ? ParsingUtils.join(";", ParsingUtils.sortList(vc.getFilters())) : (forcePASS || vc.filtersWereApplied() ? VCFConstants.PASSES_FILTERS_v4 : VCFConstants.UNFILTERED);
     }
 
     private String getQualValue(double qual) {
@@ -462,7 +419,7 @@ public class StandardVCFWriter implements VCFWriter {
         mWriter.write(encoding);
     }
 
-    private static String formatVCFField(Object val) {
+    public static String formatVCFField(Object val) {
         String result;
         if ( val == null )
             result = VCFConstants.MISSING_VALUE_v4;
@@ -524,12 +481,11 @@ public class StandardVCFWriter implements VCFWriter {
     }
 
 
-    public static int countOccurrences(char c, String s) {
+    private static int countOccurrences(char c, String s) {
            int count = 0;
            for (int i = 0; i < s.length(); i++) {
                count += s.charAt(i) == c ? 1 : 0;
            }
            return count;
     }
-
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java
index fa030ef5f..42ea05355 100755
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFCodec.java
@@ -105,34 +105,37 @@ public class VCFCodec extends AbstractVCFCodec {
      * @return a set of the filters applied or null if filters were not applied to the record (e.g. as per the missing value in a VCF)
      */
     protected Set parseFilters(String filterString) {
+        return parseFilters(filterHash, lineNo, filterString);
+    }
 
+    public static Set parseFilters(final Map> cache, final int lineNo, final String filterString) {
         // null for unfiltered
         if ( filterString.equals(VCFConstants.UNFILTERED) )
             return null;
 
-        // empty set for passes filters
-        LinkedHashSet fFields = new LinkedHashSet();
-
         if ( filterString.equals(VCFConstants.PASSES_FILTERS_v4) )
-            return fFields;
+            return Collections.emptySet();
         if ( filterString.equals(VCFConstants.PASSES_FILTERS_v3) )
-            generateException(VCFConstants.PASSES_FILTERS_v3 + " is an invalid filter name in vcf4");
+            generateException(VCFConstants.PASSES_FILTERS_v3 + " is an invalid filter name in vcf4", lineNo);
         if ( filterString.length() == 0 )
-            generateException("The VCF specification requires a valid filter status");
+            generateException("The VCF specification requires a valid filter status: filter was " + filterString, lineNo);
 
         // do we have the filter string cached?
-        if ( filterHash.containsKey(filterString) )
-            return filterHash.get(filterString);
+        if ( cache != null && cache.containsKey(filterString) )
+            return Collections.unmodifiableSet(cache.get(filterString));
 
+        // empty set for passes filters
+        LinkedHashSet fFields = new LinkedHashSet();
         // otherwise we have to parse and cache the value
         if ( filterString.indexOf(VCFConstants.FILTER_CODE_SEPARATOR) == -1 )
             fFields.add(filterString);
         else
             fFields.addAll(Arrays.asList(filterString.split(VCFConstants.FILTER_CODE_SEPARATOR)));
 
-        filterHash.put(filterString, fFields);
+        fFields = fFields;
+        if ( cache != null ) cache.put(filterString, fFields);
 
-        return fFields;
+        return Collections.unmodifiableSet(fFields);
     }
 
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
index eb01e5dca..66e11bc1e 100755
--- a/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
+++ b/public/java/src/org/broadinstitute/sting/utils/codecs/vcf/VCFHeader.java
@@ -24,6 +24,7 @@ public class VCFHeader {
     private final Set mMetaData;
     private final Map mInfoMetaData = new HashMap();
     private final Map mFormatMetaData = new HashMap();
+    private final Map mOtherMetaData = new HashMap();
 
     // the list of auxillary tags
     private final Set mGenotypeSampleNames = new LinkedHashSet();
@@ -34,9 +35,6 @@ public class VCFHeader {
     // the header string indicator
     public static final String HEADER_INDICATOR = "#";
 
-    /** do we have genotying data? */
-    private boolean hasGenotypingData = false;
-
     // were the input samples sorted originally (or are we sorting them)?
     private boolean samplesWereAlreadySorted = true;
 
@@ -56,17 +54,15 @@ public class VCFHeader {
      * create a VCF header, given a list of meta data and auxillary tags
      *
      * @param metaData            the meta data associated with this header
-     * @param genotypeSampleNames the genotype format field, and the sample names
+     * @param genotypeSampleNames the sample names
      */
     public VCFHeader(Set metaData, Set genotypeSampleNames) {
         mMetaData = new TreeSet();
         if ( metaData != null )
             mMetaData.addAll(metaData);
-        for (String col : genotypeSampleNames) {
-            if (!col.equals("FORMAT"))
-                mGenotypeSampleNames.add(col);
-        }
-        if (genotypeSampleNames.size() > 0) hasGenotypingData = true;
+
+        mGenotypeSampleNames.addAll(genotypeSampleNames);
+
         loadVCFVersion();
         loadMetaDataMaps();
 
@@ -110,6 +106,9 @@ public class VCFHeader {
                 VCFFormatHeaderLine formatLine = (VCFFormatHeaderLine)line;
                 mFormatMetaData.put(formatLine.getName(), formatLine);
             }
+            else {
+                mOtherMetaData.put(line.getKey(), line);
+            }
         }
     }
 
@@ -153,7 +152,7 @@ public class VCFHeader {
      * @return true if we have genotyping columns, false otherwise
      */
     public boolean hasGenotypingData() {
-        return hasGenotypingData;
+        return mGenotypeSampleNames.size() > 0;
     }
 
     /**
@@ -167,7 +166,7 @@ public class VCFHeader {
 
     /** @return the column count */
     public int getColumnCount() {
-        return HEADER_FIELDS.values().length + ((hasGenotypingData) ? mGenotypeSampleNames.size() + 1 : 0);
+        return HEADER_FIELDS.values().length + (hasGenotypingData() ? mGenotypeSampleNames.size() + 1 : 0);
     }
 
     /**
@@ -185,6 +184,14 @@ public class VCFHeader {
     public VCFFormatHeaderLine getFormatHeaderLine(String key) {
         return mFormatMetaData.get(key);
     }
+
+    /**
+     * @param key    the header key name
+     * @return the meta data line, or null if there is none
+     */
+    public VCFHeaderLine getOtherHeaderLine(String key) {
+        return mOtherMetaData.get(key);
+    }
 }
 
 
diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
index 274c64f42..70f7387f4 100755
--- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
+++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java
@@ -174,6 +174,12 @@ public class UserException extends ReviewedStingException {
         }
     }
 
+    public static class MalformedVCFHeader extends UserException {
+        public MalformedVCFHeader(String message) {
+            super(String.format("The provided VCF file has a malformed header: %s", message));
+        }
+    }
+
     public static class ReadMissingReadGroup extends MalformedBAM {
         public ReadMissingReadGroup(SAMRecord read) {
             super(read, String.format("Read %s is either missing the read group or its read group is not defined in the BAM header, both of which are required by the GATK.  Please use http://www.broadinstitute.org/gsa/wiki/index.php/ReplaceReadGroups to fix this problem", read.getReadName()));
diff --git a/public/java/src/org/broadinstitute/sting/utils/gcf/GCF.java b/public/java/src/org/broadinstitute/sting/utils/gcf/GCF.java
new file mode 100644
index 000000000..ef0d9ca42
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/gcf/GCF.java
@@ -0,0 +1,256 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.gcf;
+
+import org.broadinstitute.sting.utils.codecs.vcf.StandardVCFWriter;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+import org.broadinstitute.sting.utils.variantcontext.Genotype;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * GATK binary VCF record
+ *
+ * @author Your Name
+ * @since Date created
+ */
+public class GCF {
+    private final static int RECORD_TERMINATOR = 123456789;
+    private int chromOffset;
+    private int start, stop;
+    private String id;
+    private List alleleMap;
+    private int alleleOffsets[];
+    private float qual;
+    private byte refPad;
+    private String info;
+    private int filterOffset;
+
+    private List genotypes = Collections.emptyList();
+
+    public GCF(final GCFHeaderBuilder GCFHeaderBuilder, final VariantContext vc, boolean skipGenotypes) {
+        chromOffset = GCFHeaderBuilder.encodeString(vc.getChr());
+        start = vc.getStart();
+        stop = vc.getEnd();
+        refPad = vc.hasReferenceBaseForIndel() ? vc.getReferenceBaseForIndel() : 0;
+        id = vc.getID();
+
+        // encode alleles
+        alleleMap = new ArrayList(vc.getNAlleles());
+        alleleOffsets = new int[vc.getNAlleles()];
+        alleleMap.add(vc.getReference());
+        alleleOffsets[0] = GCFHeaderBuilder.encodeAllele(vc.getReference());
+        for ( int i = 0; i < vc.getAlternateAlleles().size(); i++ ) {
+            alleleMap.add(vc.getAlternateAllele(i));
+            alleleOffsets[i+1] = GCFHeaderBuilder.encodeAllele(vc.getAlternateAllele(i));
+        }
+
+        qual = (float)vc.getNegLog10PError(); //qualToByte(vc.getPhredScaledQual());
+        info = infoFieldString(vc, GCFHeaderBuilder);
+        filterOffset = GCFHeaderBuilder.encodeString(StandardVCFWriter.getFilterString(vc));
+
+        if ( ! skipGenotypes ) {
+            genotypes = encodeGenotypes(GCFHeaderBuilder, vc);
+        }
+    }
+
+    public GCF(DataInputStream inputStream, boolean skipGenotypes) throws IOException, EOFException {
+        chromOffset = inputStream.readInt();
+
+        // have we reached the footer?
+        if ( chromOffset == GCFHeader.FOOTER_START_MARKER )
+            throw new EOFException();
+
+        start = inputStream.readInt();
+        stop = inputStream.readInt();
+        id = inputStream.readUTF();
+        refPad = inputStream.readByte();
+        alleleOffsets = readIntArray(inputStream);
+        qual = inputStream.readFloat();
+        info = inputStream.readUTF();
+        filterOffset = inputStream.readInt();
+
+        int nGenotypes = inputStream.readInt();
+        int sizeOfGenotypes = inputStream.readInt();
+        if ( skipGenotypes ) {
+            genotypes = Collections.emptyList();
+            inputStream.skipBytes(sizeOfGenotypes);
+        } else {
+            genotypes = new ArrayList(nGenotypes);
+            for ( int i = 0; i < nGenotypes; i++ )
+                genotypes.add(new GCFGenotype(this, inputStream));
+        }
+
+        int recordDone = inputStream.readInt();
+        if ( recordDone != RECORD_TERMINATOR )
+            throw new UserException.MalformedFile("Record not terminated by RECORD_TERMINATOR key");
+    }
+
+    public int write(DataOutputStream outputStream) throws IOException {
+        int startSize = outputStream.size();
+        outputStream.writeInt(chromOffset);
+        outputStream.writeInt(start);
+        outputStream.writeInt(stop);
+        outputStream.writeUTF(id);
+        outputStream.writeByte(refPad);
+        writeIntArray(alleleOffsets, outputStream, true);
+        outputStream.writeFloat(qual);
+        outputStream.writeUTF(info);
+        outputStream.writeInt(filterOffset);
+
+        int nGenotypes = genotypes.size();
+        int expectedSizeOfGenotypes = nGenotypes == 0 ? 0 : genotypes.get(0).sizeInBytes() * nGenotypes;
+        outputStream.writeInt(nGenotypes);
+        outputStream.writeInt(expectedSizeOfGenotypes);
+        int obsSizeOfGenotypes = 0;
+        for ( GCFGenotype g : genotypes )
+            obsSizeOfGenotypes += g.write(outputStream);
+        if ( obsSizeOfGenotypes != expectedSizeOfGenotypes )
+            throw new RuntimeException("Expect and observed genotype sizes disagree! expect = " + expectedSizeOfGenotypes + " obs =" + obsSizeOfGenotypes);
+
+        outputStream.writeInt(RECORD_TERMINATOR);
+        return outputStream.size() - startSize;
+    }
+
+    public VariantContext decode(final String source, final GCFHeader header) {
+        final String contig = header.getString(chromOffset);
+        alleleMap = header.getAlleles(alleleOffsets);
+        double negLog10PError = qual; // QualityUtils.qualToErrorProb(qual);
+        Set filters = header.getFilters(filterOffset);
+        Map attributes = new HashMap();
+        attributes.put("INFO", info);
+        Byte refPadByte = refPad == 0 ? null : refPad;
+        Map genotypes = decodeGenotypes(header);
+
+        return new VariantContext(source, contig, start, stop, alleleMap, genotypes, negLog10PError, filters, attributes, refPadByte);
+    }
+
+    private Map decodeGenotypes(final GCFHeader header) {
+        if ( genotypes.isEmpty() )
+            return VariantContext.NO_GENOTYPES;
+        else {
+            Map map = new TreeMap();
+
+            for ( int i = 0; i < genotypes.size(); i++ ) {
+                final String sampleName = header.getSample(i);
+                final Genotype g = genotypes.get(i).decode(sampleName, header, this, alleleMap);
+                map.put(sampleName, g);
+            }
+
+            return map;
+        }
+    }
+
+    private List encodeGenotypes(final GCFHeaderBuilder GCFHeaderBuilder, final VariantContext vc) {
+        int nGenotypes = vc.getNSamples();
+        if ( nGenotypes > 0 ) {
+            List genotypes = new ArrayList(nGenotypes);
+            for ( int i = 0; i < nGenotypes; i++ ) genotypes.add(null);
+
+            for ( Genotype g : vc.getGenotypes().values() ) {
+                int i = GCFHeaderBuilder.encodeSample(g.getSampleName());
+                genotypes.set(i, new GCFGenotype(GCFHeaderBuilder, alleleMap, g));
+            }
+
+            return genotypes;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    public int getNAlleles() { return alleleOffsets.length; }
+
+
+    private final String infoFieldString(VariantContext vc, final GCFHeaderBuilder GCFHeaderBuilder) {
+        StringBuilder s = new StringBuilder();
+
+        boolean first = true;
+        for ( Map.Entry field : vc.getAttributes().entrySet() ) {
+            String key = field.getKey();
+            if ( key.equals(VariantContext.ID_KEY) || key.equals(VariantContext.UNPARSED_GENOTYPE_MAP_KEY) || key.equals(VariantContext.UNPARSED_GENOTYPE_PARSER_KEY) )
+                continue;
+            int stringIndex = GCFHeaderBuilder.encodeString(key);
+            String outputValue = StandardVCFWriter.formatVCFField(field.getValue());
+            if ( outputValue != null ) {
+                if ( ! first ) s.append(";");
+                s.append(stringIndex).append("=").append(outputValue);
+                first = false;
+            }
+        }
+
+        return s.toString();
+    }
+
+    protected final static int BUFFER_SIZE = 1048576; // 2**20
+
+    public static DataInputStream createDataInputStream(final InputStream stream) {
+        return new DataInputStream(new BufferedInputStream(stream, BUFFER_SIZE));
+    }
+
+    public static FileInputStream createFileInputStream(final File file) throws FileNotFoundException {
+        return new FileInputStream(file);
+    }
+
+    protected final static int[] readIntArray(final DataInputStream inputStream) throws IOException {
+        return readIntArray(inputStream, inputStream.readInt());
+    }
+
+    protected final static int[] readIntArray(final DataInputStream inputStream, int size) throws IOException {
+        int[] array = new int[size];
+        for ( int i = 0; i < array.length; i++ )
+            array[i] = inputStream.readInt();
+        return array;
+    }
+
+    protected final static void writeIntArray(int[] array, final DataOutputStream outputStream, boolean writeSize) throws IOException {
+        if ( writeSize ) outputStream.writeInt(array.length);
+        for ( int i : array )
+            outputStream.writeInt(i);
+    }
+
+    protected final static byte[] readByteArray(final DataInputStream inputStream) throws IOException {
+        return readByteArray(inputStream, inputStream.readInt());
+    }
+
+    protected final static byte[] readByteArray(final DataInputStream inputStream, int size) throws IOException {
+        byte[] array = new byte[size];
+        for ( int i = 0; i < array.length; i++ )
+            array[i] = inputStream.readByte();
+        return array;
+    }
+
+    protected final static void writeByteArray(byte[] array, final DataOutputStream outputStream, boolean writeSize) throws IOException {
+        if ( writeSize ) outputStream.writeInt(array.length);
+        for ( byte i : array )
+            outputStream.writeByte(i);
+    }
+
+    protected final static byte qualToByte(double phredScaledQual) {
+        return (byte)Math.round(Math.min(phredScaledQual, 255));
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/gcf/GCFGenotype.java b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFGenotype.java
new file mode 100644
index 000000000..dd1fb091c
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFGenotype.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.gcf;
+
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+import org.broadinstitute.sting.utils.variantcontext.Genotype;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * GATK binary VCF record
+ *
+ * @author Your Name
+ * @since Date created
+ */
+public class GCFGenotype {
+    private byte gq;
+    private int gt;
+    private int dp;
+    private int ad[];
+    private byte[] pl;
+
+    // todo -- what to do about phasing?  Perhaps we shouldn't support it
+    // todo -- is the FL field generic or just a flag?  Should we even support per sample filtering?
+
+    public GCFGenotype(final GCFHeaderBuilder GCFHeaderBuilder, final List allAlleles, Genotype genotype) {
+        gq = GCF.qualToByte(genotype.getPhredScaledQual());
+        gt = encodeAlleles(genotype.getAlleles(), allAlleles);
+
+        dp = genotype.getAttributeAsInt("DP", 0);
+
+        int nAlleles = allAlleles.size();
+        ad = new int[nAlleles];
+
+        int npls = nAllelesToNPls(nAlleles);
+        pl = new byte[npls];
+    }
+
+    private int nAllelesToNPls( int nAlleles ) {
+        return nAlleles*(nAlleles+1) / 2;
+    }
+
+    public GCFGenotype(GCF GCF, DataInputStream inputStream) throws IOException {
+        int gqInt = inputStream.readUnsignedByte();
+        gq = (byte)gqInt;
+        gt = inputStream.readInt();
+        dp = inputStream.readInt();
+        ad = GCF.readIntArray(inputStream, GCF.getNAlleles());
+        pl = GCF.readByteArray(inputStream, nAllelesToNPls(GCF.getNAlleles()));
+    }
+
+    // 2 alleles => 1 + 8 + 8 + 3 => 20
+    protected int sizeInBytes() {
+        return 1 // gq
+                + 4 * 2 // gt + dp
+                + 4 * ad.length // ad
+                + 1 * pl.length; // pl
+    }
+
+    public Genotype decode(final String sampleName, final GCFHeader header, GCF GCF, List alleleIndex) {
+        final List alleles = decodeAlleles(gt, alleleIndex);
+        final double negLog10PError = gq / 10.0;
+        final Set filters = Collections.emptySet();
+        final Map attributes = new HashMap();
+        attributes.put("DP", dp);
+        attributes.put("AD", ad);
+        attributes.put("PL", pl);
+
+        return new Genotype(sampleName, alleles, negLog10PError, filters, attributes, false);
+    }
+
+    private static int encodeAlleles(List gtList, List allAlleles) {
+        final int nAlleles = gtList.size();
+        if ( nAlleles  > 4 )
+            throw new IllegalArgumentException("encodeAlleles doesn't support more than 4 alt alleles, but I saw " + gtList);
+
+        int gtInt = 0;
+        for ( int i = 0; i < nAlleles ; i++ ) {
+            final int bitOffset = i * 8;
+            final int allelei = getAlleleIndex(gtList.get(i), allAlleles);
+            final int gti = (allelei + 1) << bitOffset;
+            gtInt = gtInt | gti;
+        }
+
+        return gtInt;
+    }
+
+    private static int getAlleleIndex(Allele q, List allAlleles) {
+        if ( q.isNoCall() )
+            return 254;
+        for ( int i = 0; i < allAlleles.size(); i++ )
+            if ( q.equals(allAlleles.get(i)) )
+                return i;
+        throw new IllegalStateException("getAlleleIndex passed allele not in map! allele " + q + " allAlleles " + allAlleles);
+    }
+
+    private static List decodeAlleles(int gtInt, List alleleIndex) {
+        List alleles = new ArrayList(4);
+
+        for ( int i = 0; i < 32; i += 8 ) {
+            final int gi = (gtInt & (0x000000FF << i)) >> i;
+            if ( gi != 0 ) {
+                final int allelei = gi - 1;
+                alleles.add( allelei == 254 ? Allele.NO_CALL : alleleIndex.get(allelei) );
+            } else {
+                break;
+            }
+        }
+
+        return alleles;
+    }
+
+    public int write(DataOutputStream outputStream) throws IOException {
+        int startSize = outputStream.size();
+        outputStream.writeByte(gq);
+        outputStream.writeInt(gt);
+        outputStream.writeInt(dp);
+        GCF.writeIntArray(ad, outputStream, false);
+        GCF.writeByteArray(pl, outputStream, false);
+        return outputStream.size() - startSize;
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeader.java b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeader.java
new file mode 100644
index 000000000..6d96eda56
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeader.java
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.gcf;
+
+import org.apache.log4j.Logger;
+import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
+import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * [Short one sentence description of this walker]
+ * 

+ * 

+ * [Functionality of this walker]
+ * 

+ * 

+ * 
Input

+ * 

+ * [Input description]
+ * 

+ * 

+ * 
Output

+ * 

+ * [Output description]
+ * 

+ * 

+ * 
Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T $WalkerName
+ *  

+ *
+ * @author Your Name
+ * @since Date created
+ */
+public class GCFHeader {
+    final protected static Logger logger = Logger.getLogger(GCFHeader.class);
+
+    public final static int GCF_VERSION = 1;
+    public final static byte[] GCF_FILE_START_MARKER = "GCF\1".getBytes();
+    public final static int FOOTER_START_MARKER = -1;
+    public final static long HEADER_FORWARD_REFERENCE_OFFSET = GCF_FILE_START_MARKER.length + 4; // for the version
+
+    final int version;
+    long footerPosition;
+    final List alleles;
+    final List strings;
+    final List samples;
+    final List> filters;
+
+    public GCFHeader(final Map allelesIn, final Map stringIn, final Map samplesIn) {
+        version = GCF_VERSION;
+        footerPosition = 0;
+        this.alleles = linearize(allelesIn);
+        this.strings = linearize(stringIn);
+        this.samples = linearize(samplesIn);
+        this.filters = null; // not used with this constructor
+    }
+
+    public GCFHeader(FileInputStream fileInputStream) throws IOException {
+        DataInputStream inputStream = new DataInputStream(fileInputStream);
+        byte[] headerTest = new byte[GCF_FILE_START_MARKER.length];
+        inputStream.read(headerTest);
+        if ( ! Arrays.equals(headerTest, GCF_FILE_START_MARKER) ) {
+            throw new UserException("Could not read GVCF file.  GCF_FILE_START_MARKER missing.  Saw " + new String(headerTest));
+        } else {
+            version = inputStream.readInt();
+            logger.info("Read GCF version " + version);
+            footerPosition = inputStream.readLong();
+            logger.info("Read footer position of " + footerPosition);
+            long lastPos = fileInputStream.getChannel().position();
+            logger.info("  Last position is " + lastPos);
+
+            // seek to the footer
+            fileInputStream.getChannel().position(footerPosition);
+            if ( inputStream.readInt() != FOOTER_START_MARKER )
+                throw new UserException.MalformedFile("Malformed GCF file: couldn't find the footer marker");
+            alleles = stringsToAlleles(readStrings(inputStream));
+            strings = readStrings(inputStream);
+            samples = readStrings(inputStream);
+            logger.info(String.format("Allele map of %d elements", alleles.size()));
+            logger.info(String.format("String map of %d elements", strings.size()));
+            logger.info(String.format("Sample map of %d elements", samples.size()));
+            filters = initializeFilterCache();
+            fileInputStream.getChannel().position(lastPos);
+        }
+    }
+
+    public static int writeHeader(final DataOutputStream outputStream) throws IOException {
+        int startBytes = outputStream.size();
+        outputStream.write(GCF_FILE_START_MARKER);
+        outputStream.writeInt(GCF_VERSION);
+        outputStream.writeLong(0);
+        return outputStream.size() - startBytes;
+    }
+
+    public int writeFooter(final DataOutputStream outputStream) throws IOException {
+        int startBytes = outputStream.size();
+        outputStream.writeInt(FOOTER_START_MARKER); // has to be the same as chrom encoding
+        write(outputStream, allelesToStrings(alleles));
+        write(outputStream, strings);
+        write(outputStream, samples);
+        return outputStream.size() - startBytes;
+    }
+
+    private void write(DataOutputStream outputStream, List l) throws IOException {
+        outputStream.writeInt(l.size());
+        for ( String elt : l ) outputStream.writeUTF(elt);
+    }
+
+    private List allelesToStrings(List alleles) {
+        List strings = new ArrayList(alleles.size());
+        for ( Allele allele : alleles ) strings.add(allele.toString());
+        return strings;
+    }
+
+    private List> initializeFilterCache() {
+        // required to allow offset -> set lookup
+        List> l = new ArrayList>(strings.size());
+        for ( int i = 0; i < strings.size(); i++ ) l.add(null);
+        return l;
+    }
+
+    private static List stringsToAlleles(final List strings) {
+        final List alleles = new ArrayList(strings.size());
+        for ( String string : strings ) {
+            boolean isRef = string.endsWith("*");
+            if ( isRef ) string = string.substring(0, string.length() - 1);
+            alleles.add(Allele.create(string, isRef));
+        }
+        return alleles;
+    }
+
+    private static List readStrings(final DataInputStream inputStream) throws IOException {
+        final int nStrings = inputStream.readInt();
+
+        final List strings = new ArrayList(nStrings);
+        for ( int i = 0; i < nStrings; i++ ) {
+            strings.add(inputStream.readUTF());
+        }
+
+        return strings;
+    }
+
+    private static  List linearize(final Map map) {
+        final ArrayList l = new ArrayList(map.size());
+        for ( int i = 0; i < map.size(); i++ ) l.add(null);
+        for ( final Map.Entry elt : map.entrySet() )
+            l.set(elt.getValue(), elt.getKey());
+        return l;
+    }
+
+    public String getSample(final int offset) { return samples.get(offset); }
+    public String getString(final int offset) { return strings.get(offset); }
+    public Allele getAllele(final int offset) { return alleles.get(offset); }
+    public List getAlleles(final int[] offsets) {
+        final List alleles = new ArrayList(offsets.length);
+        for ( int i : offsets ) alleles.add(getAllele(i));
+        return alleles;
+    }
+
+    public Set getFilters(final int offset) {
+        Set cached = filters.get(offset);
+
+        if ( cached != null )
+            return cached;
+        else {
+            final String filterString = getString(offset);
+            if ( filterString.equals(VCFConstants.UNFILTERED) )
+                return null; // UNFILTERED records are represented by null
+            else {
+                Set set = VCFCodec.parseFilters(null, -1, filterString);
+                filters.set(offset, set); // remember the result
+                return set;
+            }
+        }
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeaderBuilder.java b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeaderBuilder.java
new file mode 100644
index 000000000..40e01ec72
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFHeaderBuilder.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.gcf;
+
+import org.broadinstitute.sting.utils.variantcontext.Allele;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * [Short one sentence description of this walker]
+ * 

+ * 

+ * [Functionality of this walker]
+ * 

+ * 

+ * 
Input

+ * 

+ * [Input description]
+ * 

+ * 

+ * 
Output

+ * 

+ * [Output description]
+ * 

+ * 

+ * 
Examples

+ * 
+ *    java
+ *      -jar GenomeAnalysisTK.jar
+ *      -T $WalkerName
+ *  

+ *
+ * @author Your Name
+ * @since Date created
+ */
+public class GCFHeaderBuilder {
+    Map alleles = new HashMap();
+    Map strings = new HashMap();
+    Map samples = new HashMap();
+
+    public GCFHeader createHeader() {
+        return new GCFHeader(alleles, strings, samples);
+    }
+
+    public int encodeString(final String chr)    { return encode(strings, chr); }
+    public int encodeAllele(final Allele allele) { return encode(alleles, allele); }
+    public int encodeSample(final String sampleName) { return encode(samples, sampleName); }
+
+    private  int encode(Map map, T key) {
+        Integer v = map.get(key);
+        if ( v == null ) {
+            v = map.size();
+            map.put(key, v);
+        }
+        return v;
+    }
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/gcf/GCFWriter.java b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFWriter.java
new file mode 100644
index 000000000..18fae18c4
--- /dev/null
+++ b/public/java/src/org/broadinstitute/sting/utils/gcf/GCFWriter.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.gcf;
+
+import net.sf.samtools.SAMSequenceDictionary;
+import org.broadinstitute.sting.utils.codecs.vcf.IndexingVCFWriter;
+import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.variantcontext.VariantContext;
+
+import java.io.*;
+
+/**
+ * GCFWriter implementing the VCFWriter interface
+ * @author Your Name
+ * @since Date created
+ */
+public class GCFWriter extends IndexingVCFWriter {
+    final boolean skipGenotypes;
+    final FileOutputStream fileOutputStream;
+    final DataOutputStream dataOutputStream;
+    final GCFHeaderBuilder gcfHeaderBuilder;
+    int nbytes = 0;
+    VCFHeader header = null;
+    File location;
+
+    // --------------------------------------------------------------------------------
+    //
+    // Constructors
+    //
+    // --------------------------------------------------------------------------------
+
+    public GCFWriter(final File location, final SAMSequenceDictionary refDict, boolean enableOnTheFlyIndexing, boolean doNotWriteGenotypes) {
+        super(writerName(location, null), location, null, refDict, enableOnTheFlyIndexing);
+        this.location = location;
+        this.skipGenotypes = doNotWriteGenotypes;
+
+        // write the output
+        try {
+            fileOutputStream = new FileOutputStream(location);
+            dataOutputStream = createDataOutputStream(fileOutputStream);
+            gcfHeaderBuilder = new GCFHeaderBuilder();
+        } catch ( FileNotFoundException e ) {
+            throw new UserException.CouldNotCreateOutputFile(location, e);
+        }
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // VCFWriter interface functions
+    //
+    // --------------------------------------------------------------------------------
+
+    @Override
+    public void writeHeader(VCFHeader header) {
+        this.header = header;
+        try {
+            nbytes += GCFHeader.writeHeader(dataOutputStream);
+        } catch ( IOException e ) {
+            throw new UserException.CouldNotCreateOutputFile(getStreamName(), "Couldn't write header", e);
+        }
+    }
+
+    @Override
+    public void add(VariantContext vc) {
+        super.add(vc);
+        GCF gcf = new GCF(gcfHeaderBuilder, vc, skipGenotypes);
+        try {
+            nbytes += gcf.write(dataOutputStream);
+        } catch ( IOException e ) {
+            throw new UserException.CouldNotCreateOutputFile(getStreamName(), "Failed to add gcf record " + gcf + " to stream " + getStreamName(), e);
+        }
+    }
+
+    @Override
+    public void close() {
+        // todo -- write out VCF header lines
+        GCFHeader gcfHeader = gcfHeaderBuilder.createHeader();
+        try {
+            long headerPosition = nbytes;
+            nbytes += gcfHeader.writeFooter(dataOutputStream);
+            dataOutputStream.close();
+            //System.out.println("Writing forward reference to " + headerPosition);
+
+            RandomAccessFile raFile = new RandomAccessFile(location, "rw");
+            raFile.seek(GCFHeader.HEADER_FORWARD_REFERENCE_OFFSET);
+            raFile.writeLong(headerPosition);
+            raFile.close();
+        } catch ( IOException e ) {
+            throw new ReviewedStingException("Failed to close GCFWriter " + getStreamName(), e);
+        }
+
+        super.close();
+    }
+
+    private static final DataOutputStream createDataOutputStream(final OutputStream stream) {
+        return new DataOutputStream(new BufferedOutputStream(stream, GCF.BUFFER_SIZE));
+    }
+
+}
diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java
index 4f1e95499..7ea77a939 100644
--- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java
+++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java
@@ -66,7 +66,8 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
     public boolean includeInDocs(ClassDoc doc) {
         try {
             Class type = HelpUtils.getClassForDoc(doc);
-            return JVMUtils.isConcrete(type);
+            boolean hidden = ! getDoclet().showHiddenFeatures() && type.isAnnotationPresent(Hidden.class);
+            return ! hidden && JVMUtils.isConcrete(type);
         } catch ( ClassNotFoundException e ) {
             return false;
         }
diff --git a/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java b/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java
index f551e1368..41cbbe59f 100644
--- a/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/interval/IntervalUtils.java
@@ -334,24 +334,44 @@ public class IntervalUtils {
     }
 
     /**
-     * Splits an interval list into multiple files.
-     * @param fileHeader The sam file header.
+     * Splits an interval list into multiple sublists.
      * @param locs The genome locs to split.
      * @param splits The stop points for the genome locs returned by splitFixedIntervals.
-     * @param scatterParts The output interval lists to write to.
+     * @return A list of lists of genome locs, split according to splits
      */
-    public static void scatterFixedIntervals(SAMFileHeader fileHeader, List locs, List splits, List scatterParts) {
-        if (splits.size() != scatterParts.size())
-            throw new UserException.BadArgumentValue("splits", String.format("Split points %d does not equal the number of scatter parts %d.", splits.size(), scatterParts.size()));
-        int fileIndex = 0;
+    public static List> splitIntervalsToSubLists(List locs, List splits) {
         int locIndex = 1;
         int start = 0;
+        List> sublists = new ArrayList>(splits.size());
         for (Integer stop: splits) {
-            IntervalList intervalList = new IntervalList(fileHeader);
+            List curList = new ArrayList();
             for (int i = start; i < stop; i++)
-                intervalList.add(toInterval(locs.get(i), locIndex++));
-            intervalList.write(scatterParts.get(fileIndex++));
+                curList.add(locs.get(i));
             start = stop;
+            sublists.add(curList);
+        }
+
+        return sublists;
+    }
+
+
+    /**
+     * Splits an interval list into multiple files.
+     * @param fileHeader The sam file header.
+     * @param splits Pre-divided genome locs returned by splitFixedIntervals.
+     * @param scatterParts The output interval lists to write to.
+     */
+    public static void scatterFixedIntervals(SAMFileHeader fileHeader, List> splits, List scatterParts) {
+        if (splits.size() != scatterParts.size())
+            throw new UserException.BadArgumentValue("splits", String.format("Split points %d does not equal the number of scatter parts %d.", splits.size(), scatterParts.size()));
+
+        int fileIndex = 0;
+        int locIndex = 1;
+        for (final List split : splits) {
+            IntervalList intervalList = new IntervalList(fileHeader);
+            for (final GenomeLoc loc : split)
+                intervalList.add(toInterval(loc, locIndex++));
+            intervalList.write(scatterParts.get(fileIndex++));
         }
     }
 
@@ -361,17 +381,15 @@ public class IntervalUtils {
      * @param numParts Number of parts to split the locs into.
      * @return The stop points to split the genome locs.
      */
-    public static List splitFixedIntervals(List locs, int numParts) {
+    public static List> splitFixedIntervals(List locs, int numParts) {
         if (locs.size() < numParts)
             throw new UserException.BadArgumentValue("scatterParts", String.format("Cannot scatter %d locs into %d parts.", locs.size(), numParts));
-        long locsSize = 0;
-        for (GenomeLoc loc: locs)
-            locsSize += loc.size();
-        List splitPoints = new ArrayList();
+        final long locsSize = intervalSize(locs);
+        final List splitPoints = new ArrayList();
         addFixedSplit(splitPoints, locs, locsSize, 0, locs.size(), numParts);
         Collections.sort(splitPoints);
         splitPoints.add(locs.size());
-        return splitPoints;
+        return splitIntervalsToSubLists(locs, splitPoints);
     }
 
     private static void addFixedSplit(List splitPoints, List locs, long locsSize, int startIndex, int stopIndex, int numParts) {
@@ -441,4 +459,11 @@ public class IntervalUtils {
             return merged;
         }
     }
+
+    public static final long intervalSize(final List locs) {
+        long size = 0;
+        for ( final GenomeLoc loc : locs )
+            size += loc.size();
+        return size;
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java
index 12899e898..053864791 100755
--- a/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java
+++ b/public/java/src/org/broadinstitute/sting/utils/pileup/PileupElement.java
@@ -81,20 +81,17 @@ public class PileupElement {
     //
     // --------------------------------------------------------------------------
 
-    private Integer getReducedReadQualityTagValue() {
-        return getRead().getIntegerAttribute(ReadUtils.REDUCED_READ_QUALITY_TAG);
-    }
-
     public boolean isReducedRead() {
-        return getReducedReadQualityTagValue() != null;
+        return ReadUtils.isReducedRead(getRead());
     }
 
     public int getReducedCount() {
+        if ( ! isReducedRead() ) throw new IllegalArgumentException("Cannot get reduced count for non-reduced read " + getRead().getReadName());
         return (int)getQual();
     }
 
     public byte getReducedQual() {
-        return (byte)(int)getReducedReadQualityTagValue();
+        return (byte)(int)ReadUtils.getReducedReadQualityTagValue(getRead());
     }
 
 }
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java
old mode 100644
new mode 100755
index 62bbb0307..8beb1b21a
--- a/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/sam/ReadUtils.java
@@ -43,10 +43,43 @@ import java.util.*;
  * @version 0.1
  */
 public class ReadUtils {
-    public static final String REDUCED_READ_QUALITY_TAG = "RQ";
-
     private ReadUtils() { }
 
+    // ----------------------------------------------------------------------------------------------------
+    //
+    // Reduced read utilities
+    //
+    // ----------------------------------------------------------------------------------------------------
+
+    public static final String REDUCED_READ_QUALITY_TAG = "RQ";
+
+    public final static Integer getReducedReadQualityTagValue(final SAMRecord read) {
+        return read.getIntegerAttribute(ReadUtils.REDUCED_READ_QUALITY_TAG);
+    }
+
+    public final static boolean isReducedRead(final SAMRecord read) {
+        return getReducedReadQualityTagValue(read) != null;
+    }
+
+    public final static SAMRecord reducedReadWithReducedQuals(final SAMRecord read) {
+        if ( ! isReducedRead(read) ) throw new IllegalArgumentException("read must be a reduced read");
+        try {
+            SAMRecord newRead = (SAMRecord)read.clone();
+            byte reducedQual = (byte)(int)getReducedReadQualityTagValue(read);
+            byte[] newQuals = new byte[read.getBaseQualities().length];
+            Arrays.fill(newQuals, reducedQual);
+            newRead.setBaseQualities(newQuals);
+            return newRead;
+        } catch ( CloneNotSupportedException e ) {
+            throw new ReviewedStingException("SAMRecord no longer supports clone", e);
+        }
+    }
+
+    // ----------------------------------------------------------------------------------------------------
+    //
+    // General utilities
+    //
+    // ----------------------------------------------------------------------------------------------------
     public static SAMFileHeader copySAMFileHeader(SAMFileHeader toCopy) {
         SAMFileHeader copy = new SAMFileHeader();
 
@@ -118,37 +151,46 @@ public class ReadUtils {
     /**
      * This enum represents all the different ways in which a read can overlap an interval.
      *
-     * NO_OVERLAP:
+     * NO_OVERLAP_CONTIG:
+     * read and interval are in different contigs.
+     *
+     * NO_OVERLAP_LEFT:
+     * the read does not overlap the interval.
+     *
+     *                        |----------------| (interval)
+     *   <---------------->                      (read)
+     *
+     * NO_OVERLAP_RIGHT:
      * the read does not overlap the interval.
      *
      *   |----------------|                      (interval)
      *                        <----------------> (read)
      *
-     * LEFT_OVERLAP:
+     * OVERLAP_LEFT:
      * the read starts before the beginning of the interval but ends inside of it
      *
      *          |----------------| (interval)
      *   <---------------->        (read)
      *
-     * RIGHT_OVERLAP:
+     * OVERLAP_RIGHT:
      * the read starts inside the interval but ends outside of it
      *
      *   |----------------|     (interval)
      *       <----------------> (read)
      *
-     * FULL_OVERLAP:
+     * OVERLAP_LEFT_AND_RIGHT:
      * the read starts before the interval and ends after the interval
      *
      *      |-----------|     (interval)
      *  <-------------------> (read)
      *
-     * CONTAINED:
+     * OVERLAP_CONTAINED:
      * the read starts and ends inside the interval
      *
      *  |----------------|     (interval)
      *     <-------->          (read)
      */
-    public enum ReadAndIntervalOverlap {NO_OVERLAP_CONTIG, NO_OVERLAP_LEFT, NO_OVERLAP_RIGHT, OVERLAP_LEFT, OVERLAP_RIGHT, OVERLAP_LEFT_AND_RIGHT, OVERLAP_CONTAINED}
+    public enum ReadAndIntervalOverlap {NO_OVERLAP_CONTIG, NO_OVERLAP_LEFT, NO_OVERLAP_RIGHT, NO_OVERLAP_HARDCLIPPED_LEFT, NO_OVERLAP_HARDCLIPPED_RIGHT, OVERLAP_LEFT, OVERLAP_RIGHT, OVERLAP_LEFT_AND_RIGHT, OVERLAP_CONTAINED}
 
     /**
      * God, there's a huge information asymmetry in SAM format:
@@ -631,34 +673,42 @@ public class ReadUtils {
      */
     public static ReadAndIntervalOverlap getReadAndIntervalOverlapType(SAMRecord read, GenomeLoc interval) {
 
-        int start = getRefCoordSoftUnclippedStart(read);
-        int stop = getRefCoordSoftUnclippedEnd(read);
+        int sStart = getRefCoordSoftUnclippedStart(read);
+        int sStop = getRefCoordSoftUnclippedEnd(read);
+        int uStart = read.getUnclippedStart();
+        int uStop = read.getUnclippedEnd();
 
         if ( !read.getReferenceName().equals(interval.getContig()) )
             return ReadAndIntervalOverlap.NO_OVERLAP_CONTIG;
 
-        else if  ( stop < interval.getStart() )
+        else if ( uStop < interval.getStart() )
             return ReadAndIntervalOverlap.NO_OVERLAP_LEFT;
 
-        else if ( start > interval.getStop() )
+        else if ( uStart > interval.getStop() )
             return ReadAndIntervalOverlap.NO_OVERLAP_RIGHT;
 
-        else if ( (start >= interval.getStart()) &&
-                  (stop <= interval.getStop()) )
+        else if ( sStop < interval.getStart() )
+            return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_LEFT;
+
+        else if ( sStart > interval.getStop() )
+            return ReadAndIntervalOverlap.NO_OVERLAP_HARDCLIPPED_RIGHT;
+
+        else if ( (sStart >= interval.getStart()) &&
+                  (sStop <= interval.getStop()) )
             return ReadAndIntervalOverlap.OVERLAP_CONTAINED;
 
-        else if ( (start < interval.getStart()) &&
-                  (stop > interval.getStop()) )
+        else if ( (sStart < interval.getStart()) &&
+                  (sStop > interval.getStop()) )
             return ReadAndIntervalOverlap.OVERLAP_LEFT_AND_RIGHT;
 
-        else if ( (start < interval.getStart()) )
+        else if ( (sStart < interval.getStart()) )
             return ReadAndIntervalOverlap.OVERLAP_LEFT;
 
         else
             return ReadAndIntervalOverlap.OVERLAP_RIGHT;
     }
 
-    @Ensures({"result >= read.getUnclippedStart()", "result <= read.getUnclippedEnd()"})
+    @Ensures({"result >= read.getUnclippedStart()", "result <= read.getUnclippedEnd() || readIsEntirelyInsertion(read)"})
     public static int getRefCoordSoftUnclippedStart(SAMRecord read) {
         int start = read.getUnclippedStart();
         for (CigarElement cigarElement : read.getCigar().getCigarElements()) {
@@ -670,9 +720,13 @@ public class ReadUtils {
         return start;
     }
 
-    @Ensures({"result >= read.getUnclippedStart()", "result <= read.getUnclippedEnd()"})
+    @Ensures({"result >= read.getUnclippedStart()", "result <= read.getUnclippedEnd() || readIsEntirelyInsertion(read)"})
     public static int getRefCoordSoftUnclippedEnd(SAMRecord read) {
         int stop = read.getUnclippedStart();
+
+        if (readIsEntirelyInsertion(read))
+            return stop;
+
         int shift = 0;
         CigarOperator lastOperator = null;
         for (CigarElement cigarElement : read.getCigar().getCigarElements()) {
@@ -686,6 +740,14 @@ public class ReadUtils {
         return (lastOperator == CigarOperator.HARD_CLIP) ? stop-1 : stop+shift-1 ;
     }
 
+    private static boolean readIsEntirelyInsertion(SAMRecord read) {
+        for (CigarElement cigarElement : read.getCigar().getCigarElements()) {
+            if (cigarElement.getOperator() != CigarOperator.INSERTION)
+                return false;
+        }
+        return true;
+    }
+
     /**
      * Looks for a read coordinate that corresponds to the reference coordinate in the soft clipped region before
      * the alignment start of the read.
@@ -766,5 +828,47 @@ public class ReadUtils {
         return readBases;
     }
 
+    public static SAMRecord unclipSoftClippedBases(SAMRecord rec) {
+        int newReadStart = rec.getAlignmentStart();
+        int newReadEnd = rec.getAlignmentEnd();
+        List newCigarElements = new ArrayList(rec.getCigar().getCigarElements().size());
+        int heldOver = -1;
+        boolean sSeen = false;
+        for ( CigarElement e : rec.getCigar().getCigarElements() ) {
+            if ( e.getOperator().equals(CigarOperator.S) ) {
+                newCigarElements.add(new CigarElement(e.getLength(),CigarOperator.M));
+                if ( sSeen ) {
+                    newReadEnd += e.getLength();
+                    sSeen = true;
+                } else {
+                    newReadStart -= e.getLength();
+                }
+            } else {
+                newCigarElements.add(e);
+            }
+        }
+        // merge duplicate operators together
+        int idx = 0;
+        List finalCigarElements = new ArrayList(rec.getCigar().getCigarElements().size());
+        while ( idx < newCigarElements.size() -1 ) {
+            if ( newCigarElements.get(idx).getOperator().equals(newCigarElements.get(idx+1).getOperator()) ) {
+                int combSize = newCigarElements.get(idx).getLength();
+                int offset = 0;
+                while (  idx + offset < newCigarElements.size()-1 && newCigarElements.get(idx+offset).getOperator().equals(newCigarElements.get(idx+1+offset).getOperator()) ) {
+                    combSize += newCigarElements.get(idx+offset+1).getLength();
+                    offset++;
+                }
+                finalCigarElements.add(new CigarElement(combSize,newCigarElements.get(idx).getOperator()));
+                idx = idx + offset -1;
+            } else {
+                finalCigarElements.add(newCigarElements.get(idx));
+            }
+            idx++;
+        }
 
+        rec.setCigar(new Cigar(finalCigarElements));
+        rec.setAlignmentStart(newReadStart);
+
+        return rec;
+    }
 }
diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java
index fdf3d97db..7ab3f81f0 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/Genotype.java
@@ -26,14 +26,31 @@ public class Genotype {
     protected boolean filtersWereAppliedToContext;
 
     public Genotype(String sampleName, List alleles, double negLog10PError, Set filters, Map attributes, boolean isPhased) {
+        this(sampleName, alleles, negLog10PError, filters, attributes, isPhased, null);
+    }
+
+    public Genotype(String sampleName, List alleles, double negLog10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) {
         if ( alleles != null )
             this.alleles = Collections.unmodifiableList(alleles);
         commonInfo = new InferredGeneticContext(sampleName, negLog10PError, filters, attributes);
+        if ( log10Likelihoods != null )
+            commonInfo.putAttribute(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY, GenotypeLikelihoods.fromLog10Likelihoods(log10Likelihoods));
         filtersWereAppliedToContext = filters != null;
         this.isPhased = isPhased;
         validate();
     }
 
+    /**
+     * Creates a new Genotype for sampleName with genotype according to alleles.
+     * @param sampleName
+     * @param alleles
+     * @param negLog10PError the confidence in these alleles
+     * @param log10Likelihoods a log10 likelihoods for each of the genotype combinations possible for alleles, in the standard VCF ordering, or null if not known
+     */
+    public Genotype(String sampleName, List alleles, double negLog10PError, double[] log10Likelihoods) {
+        this(sampleName, alleles, negLog10PError, null, null, false, log10Likelihoods);
+    }
+
     public Genotype(String sampleName, List alleles, double negLog10PError) {
         this(sampleName, alleles, negLog10PError, null, null, false);
     }
@@ -57,13 +74,6 @@ public class Genotype {
         return new Genotype(g.getSampleName(), g.getAlleles(), g.getNegLog10PError(), g.filtersWereApplied() ? g.getFilters() : null, attributes, g.isPhased());
     }
 
-    public static Genotype removePLs(Genotype g) {
-        Map attrs = new HashMap(g.getAttributes());
-        attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY);
-        attrs.remove(VCFConstants.GENOTYPE_LIKELIHOODS_KEY);
-        return new Genotype(g.getSampleName(), g.getAlleles(), g.getNegLog10PError(), g.filtersWereApplied() ? g.getFilters() : null, attrs, g.isPhased());
-    }
-
     public static Genotype modifyAlleles(Genotype g, List alleles) {
         return new Genotype(g.getSampleName(), alleles, g.getNegLog10PError(), g.filtersWereApplied() ? g.getFilters() : null, g.getAttributes(), g.isPhased());
     }
@@ -258,7 +268,8 @@ public class Genotype {
      * @param  the value type
      * @return a sting, enclosed in {}, with comma seperated key value pairs in order of the keys
      */
-    public static , V> String sortedString(Map c) {
+    private static , V> String sortedString(Map c) {
+        // NOTE -- THIS IS COPIED FROM GATK UTILS TO ALLOW US TO KEEP A SEPARATION BETWEEN THE GATK AND VCF CODECS
         List t = new ArrayList(c.keySet());
         Collections.sort(t);
 
@@ -293,17 +304,8 @@ public class Genotype {
         return commonInfo.getAttribute(key, defaultValue); 
     }
 
-    public String getAttributeAsString(String key)                        { return commonInfo.getAttributeAsString(key); }
     public String getAttributeAsString(String key, String defaultValue)   { return commonInfo.getAttributeAsString(key, defaultValue); }
-    public int getAttributeAsInt(String key)                              { return commonInfo.getAttributeAsInt(key); }
     public int getAttributeAsInt(String key, int defaultValue)            { return commonInfo.getAttributeAsInt(key, defaultValue); }
-    public double getAttributeAsDouble(String key)                        { return commonInfo.getAttributeAsDouble(key); }
     public double getAttributeAsDouble(String key, double  defaultValue)  { return commonInfo.getAttributeAsDouble(key, defaultValue); }
-    public boolean getAttributeAsBoolean(String key)                        { return commonInfo.getAttributeAsBoolean(key); }
     public boolean getAttributeAsBoolean(String key, boolean  defaultValue)  { return commonInfo.getAttributeAsBoolean(key, defaultValue); }
-
-    public Integer getAttributeAsIntegerNoException(String key)  { return commonInfo.getAttributeAsIntegerNoException(key); }
-    public Double getAttributeAsDoubleNoException(String key)    { return commonInfo.getAttributeAsDoubleNoException(key); }
-    public String getAttributeAsStringNoException(String key)    { return commonInfo.getAttributeAsStringNoException(key); }
-    public Boolean getAttributeAsBooleanNoException(String key)  { return commonInfo.getAttributeAsBooleanNoException(key); }
 }
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/InferredGeneticContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/InferredGeneticContext.java
index 3d162adb0..bf16cd1cf 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/InferredGeneticContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/InferredGeneticContext.java
@@ -1,6 +1,8 @@
 package org.broadinstitute.sting.utils.variantcontext;
 
 
+import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
+
 import java.util.*;
 
 
@@ -204,27 +206,40 @@ public final class InferredGeneticContext {
             return defaultValue;
     }
 
-//    public AttributedObject getAttributes(Collection keys) {
-//        AttributedObject selected = new AttributedObject();
-//
-//        for ( Object key : keys )
-//            selected.putAttribute(key, this.getAttribute(key));
-//
-//        return selected;
-//    }
+    public String getAttributeAsString(String key, String defaultValue) {
+        Object x = getAttribute(key);
+        if ( x == null ) return defaultValue;
+        if ( x instanceof String ) return (String)x;
+        return String.valueOf(x); // throws an exception if this isn't a string
+    }
 
-    public String getAttributeAsString(String key)      { return (String.valueOf(getAttribute(key))); } // **NOTE**: will turn a null Object into the String "null"
-    public int getAttributeAsInt(String key)            { Object x = getAttribute(key); return x instanceof Integer ? (Integer)x : Integer.valueOf((String)x); }
-    public double getAttributeAsDouble(String key)      { Object x = getAttribute(key); return x instanceof Double ? (Double)x : Double.valueOf((String)x); }
-    public boolean getAttributeAsBoolean(String key)      { Object x = getAttribute(key); return x instanceof Boolean ? (Boolean)x : Boolean.valueOf((String)x); }
+    public int getAttributeAsInt(String key, int defaultValue) {
+        Object x = getAttribute(key);
+        if ( x == null || x == VCFConstants.MISSING_VALUE_v4 ) return defaultValue;
+        if ( x instanceof Integer ) return (Integer)x;
+        return Integer.valueOf((String)x); // throws an exception if this isn't a string
+    }
 
-    public String getAttributeAsString(String key, String defaultValue)   { return (String)getAttribute(key, defaultValue); }
-    public int getAttributeAsInt(String key, int defaultValue)            { return (Integer)getAttribute(key, defaultValue); }
-    public double getAttributeAsDouble(String key, double defaultValue)   { return (Double)getAttribute(key, defaultValue); }
-    public boolean getAttributeAsBoolean(String key, boolean defaultValue){ return (Boolean)getAttribute(key, defaultValue); }
+    public double getAttributeAsDouble(String key, double defaultValue) {
+        Object x = getAttribute(key);
+        if ( x == null ) return defaultValue;
+        if ( x instanceof Double ) return (Double)x;
+        return Double.valueOf((String)x); // throws an exception if this isn't a string
+    }
 
-    public Integer getAttributeAsIntegerNoException(String key)  { try {return getAttributeAsInt(key);} catch (Exception e) {return null;} }
-    public Double getAttributeAsDoubleNoException(String key)    { try {return getAttributeAsDouble(key);} catch (Exception e) {return null;} }
-    public String getAttributeAsStringNoException(String key)    { if (getAttribute(key) == null) return null; return getAttributeAsString(key); }
-    public Boolean getAttributeAsBooleanNoException(String key)  { try {return getAttributeAsBoolean(key);} catch (Exception e) {return null;} }
+    public boolean getAttributeAsBoolean(String key, boolean defaultValue) {
+        Object x = getAttribute(key);
+        if ( x == null ) return defaultValue;
+        if ( x instanceof Boolean ) return (Boolean)x;
+        return Boolean.valueOf((String)x); // throws an exception if this isn't a string
+    }
+
+//    public String getAttributeAsString(String key)      { return (String.valueOf(getAttribute(key))); } // **NOTE**: will turn a null Object into the String "null"
+//    public int getAttributeAsInt(String key)            { Object x = getAttribute(key); return x instanceof Integer ? (Integer)x : Integer.valueOf((String)x); }
+//    public double getAttributeAsDouble(String key)      { Object x = getAttribute(key); return x instanceof Double ? (Double)x : Double.valueOf((String)x); }
+//    public boolean getAttributeAsBoolean(String key)      { Object x = getAttribute(key); return x instanceof Boolean ? (Boolean)x : Boolean.valueOf((String)x); }
+//    public Integer getAttributeAsIntegerNoException(String key)  { try {return getAttributeAsInt(key);} catch (Exception e) {return null;} }
+//    public Double getAttributeAsDoubleNoException(String key)    { try {return getAttributeAsDouble(key);} catch (Exception e) {return null;} }
+//    public String getAttributeAsStringNoException(String key)    { if (getAttribute(key) == null) return null; return getAttributeAsString(key); }
+//    public Boolean getAttributeAsBooleanNoException(String key)  { try {return getAttributeAsBoolean(key);} catch (Exception e) {return null;} }
 }
\ No newline at end of file
diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
index 699133e38..412cbd90b 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContext.java
@@ -666,21 +666,11 @@ public class VariantContext implements Feature { // to enable tribble intergrati
         return commonInfo.getAttribute(key, defaultValue);
     }
 
-    public String getAttributeAsString(String key)                        { return commonInfo.getAttributeAsString(key); }
     public String getAttributeAsString(String key, String defaultValue)   { return commonInfo.getAttributeAsString(key, defaultValue); }
-    public int getAttributeAsInt(String key)                              { return commonInfo.getAttributeAsInt(key); }
     public int getAttributeAsInt(String key, int defaultValue)            { return commonInfo.getAttributeAsInt(key, defaultValue); }
-    public double getAttributeAsDouble(String key)                        { return commonInfo.getAttributeAsDouble(key); }
     public double getAttributeAsDouble(String key, double  defaultValue)  { return commonInfo.getAttributeAsDouble(key, defaultValue); }
-    public boolean getAttributeAsBoolean(String key)                        { return commonInfo.getAttributeAsBoolean(key); }
     public boolean getAttributeAsBoolean(String key, boolean  defaultValue)  { return commonInfo.getAttributeAsBoolean(key, defaultValue); }
 
-    public Integer getAttributeAsIntegerNoException(String key)  { return commonInfo.getAttributeAsIntegerNoException(key); }
-    public Double getAttributeAsDoubleNoException(String key)    { return commonInfo.getAttributeAsDoubleNoException(key); }
-    public String getAttributeAsStringNoException(String key)    { return commonInfo.getAttributeAsStringNoException(key); }
-    public Boolean getAttributeAsBooleanNoException(String key)  { return commonInfo.getAttributeAsBooleanNoException(key); }
-
-
     // ---------------------------------------------------------------------------------------------------------
     //
     // Working with alleles
@@ -817,6 +807,28 @@ public class VariantContext implements Feature { // to enable tribble intergrati
         throw new IllegalArgumentException("Requested " + i + " alternative allele but there are only " + n + " alternative alleles " + this);
     }
 
+    /**
+     * @param  other  VariantContext whose alternate alleles to compare against
+     * @return true if this VariantContext has the same alternate alleles as other,
+     *         regardless of ordering. Otherwise returns false.
+     */
+    public boolean hasSameAlternateAllelesAs ( VariantContext other ) {
+        Set thisAlternateAlleles = getAlternateAlleles();
+        Set otherAlternateAlleles = other.getAlternateAlleles();
+
+        if ( thisAlternateAlleles.size() != otherAlternateAlleles.size() ) {
+            return false;
+        }
+
+        for ( Allele allele : thisAlternateAlleles ) {
+            if ( ! otherAlternateAlleles.contains(allele) ) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
     // ---------------------------------------------------------------------------------------------------------
     //
     // Working with genotypes
@@ -1085,14 +1097,15 @@ public class VariantContext implements Feature { // to enable tribble intergrati
     }
 
     public void validateReferenceBases(Allele reference, Byte paddedRefBase) {
-        // don't validate if we're an insertion or complex event
-        if ( !reference.isNull() && getReference().length() == 1 && !reference.basesMatch(getReference()) ) {
-            throw new TribbleException.InternalCodecException(String.format("the REF allele is incorrect for the record at position %s:%d, %s vs. %s", getChr(), getStart(), reference.getBaseString(), getReference().getBaseString()));
+        // don't validate if we're a complex event
+        if ( !isComplexIndel() && !reference.isNull() && !reference.basesMatch(getReference()) ) {
+            throw new TribbleException.InternalCodecException(String.format("the REF allele is incorrect for the record at position %s:%d, fasta says %s vs. VCF says %s", getChr(), getStart(), reference.getBaseString(), getReference().getBaseString()));
         }
 
         // we also need to validate the padding base for simple indels
-        if ( hasReferenceBaseForIndel() && !getReferenceBaseForIndel().equals(paddedRefBase) )
-            throw new TribbleException.InternalCodecException(String.format("the padded REF base is incorrect for the record at position %s:%d, %s vs. %s", getChr(), getStart(), (char)getReferenceBaseForIndel().byteValue(), (char)paddedRefBase.byteValue()));
+        if ( hasReferenceBaseForIndel() && !getReferenceBaseForIndel().equals(paddedRefBase) ) {
+            throw new TribbleException.InternalCodecException(String.format("the padded REF base is incorrect for the record at position %s:%d, fasta says %s vs. VCF says %s", getChr(), getStart(), (char)paddedRefBase.byteValue(), (char)getReferenceBaseForIndel().byteValue()));
+        }
     }
 
     public void validateRSIDs(Set rsIDs) {
@@ -1482,7 +1495,7 @@ public class VariantContext implements Feature { // to enable tribble intergrati
 
             // Do not change the filter state if filters were not applied to this context
             Set inputVCFilters = inputVC.filtersWereAppliedToContext ? inputVC.getFilters() : null;
-            return new VariantContext(inputVC.getSource(), inputVC.getChr(), inputVC.getStart(), inputVC.getEnd(), alleles, genotypes, inputVC.getNegLog10PError(), inputVCFilters, inputVC.getAttributes());
+            return new VariantContext(inputVC.getSource(), inputVC.getChr(), inputVC.getStart(), inputVC.getEnd(), alleles, genotypes, inputVC.getNegLog10PError(), inputVCFilters, inputVC.getAttributes(),refByte);
         }
         else
             return inputVC;
diff --git a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java
index 986d6305c..ae648d547 100755
--- a/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/variantcontext/VariantContextUtils.java
@@ -44,6 +44,11 @@ import java.io.Serializable;
 import java.util.*;
 
 public class VariantContextUtils {
+    public final static String MERGE_INTERSECTION = "Intersection";
+    public final static String MERGE_FILTER_IN_ALL = "FilteredInAll";
+    public final static String MERGE_REF_IN_ALL = "ReferenceInAll";
+    public final static String MERGE_FILTER_PREFIX = "filterIn";
+
     final public static JexlEngine engine = new JexlEngine();
     static {
         engine.setSilent(false); // will throw errors now for selects that don't evaluate properly
@@ -154,6 +159,13 @@ public class VariantContextUtils {
         return "%." + precision + "f";
     }
 
+    public static Genotype removePLs(Genotype g) {
+        Map attrs = new HashMap(g.getAttributes());
+        attrs.remove(VCFConstants.PHRED_GENOTYPE_LIKELIHOODS_KEY);
+        attrs.remove(VCFConstants.GENOTYPE_LIKELIHOODS_KEY);
+        return new Genotype(g.getSampleName(), g.getAlleles(), g.getNegLog10PError(), g.filtersWereApplied() ? g.getFilters() : null, attrs, g.isPhased());
+    }
+
     /**
      * A simple but common wrapper for matching VariantContext objects using JEXL expressions
      */
@@ -316,19 +328,22 @@ public class VariantContextUtils {
         return pruneVariantContext(vc, null);
     }
 
-    public static VariantContext pruneVariantContext(VariantContext vc, Collection keysToPreserve ) {
-        MutableVariantContext mvc = new MutableVariantContext(vc);
+    public static VariantContext pruneVariantContext(final VariantContext vc, final Collection keysToPreserve ) {
+        final MutableVariantContext mvc = new MutableVariantContext(vc);
 
         if ( keysToPreserve == null || keysToPreserve.size() == 0 )
             mvc.clearAttributes();
         else {
-            Map d = mvc.getAttributes();
+            final Map d = mvc.getAttributes();
             mvc.clearAttributes();
             for ( String key : keysToPreserve )
                 if ( d.containsKey(key) )
                     mvc.putAttribute(key, d.get(key));
         }
 
+        // this must be done as the ID is stored in the attributes field
+        if ( vc.hasID() ) mvc.setID(vc.getID());
+
         Collection gs = mvc.getGenotypes().values();
         mvc.clearGenotypes();
         for ( Genotype g : gs ) {
@@ -443,34 +458,6 @@ public class VariantContextUtils {
         throw new ReviewedStingException(String.format("Couldn't find master VCF %s at %s", masterName, unsortedVCs.iterator().next()));
     }
 
-
-    public static VariantContext simpleMerge(GenomeLocParser genomeLocParser, Collection unsortedVCs, byte refBase) {
-        return simpleMerge(genomeLocParser, unsortedVCs, null, FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED, GenotypeMergeType.UNSORTED, false, false, refBase);
-    }
-
-
-    /**
-     * Merges VariantContexts into a single hybrid.  Takes genotypes for common samples in priority order, if provided.
-     * If uniqifySamples is true, the priority order is ignored and names are created by concatenating the VC name with
-     * the sample name
-     *
-     * @param genomeLocParser           loc parser
-     * @param unsortedVCs               collection of unsorted VCs
-     * @param priorityListOfVCs         priority list detailing the order in which we should grab the VCs
-     * @param filteredRecordMergeType   merge type for filtered records
-     * @param genotypeMergeOptions      merge option for genotypes
-     * @param annotateOrigin            should we annotate the set it came from?
-     * @param printMessages             should we print messages?
-     * @param inputRefBase              the ref base
-     * @return new VariantContext
-     */
-    public static VariantContext simpleMerge(GenomeLocParser genomeLocParser, Collection unsortedVCs, List priorityListOfVCs,
-                                             FilteredRecordMergeType filteredRecordMergeType, GenotypeMergeType genotypeMergeOptions,
-                                             boolean annotateOrigin, boolean printMessages, byte inputRefBase ) {
-
-        return simpleMerge(genomeLocParser, unsortedVCs, priorityListOfVCs, filteredRecordMergeType, genotypeMergeOptions, annotateOrigin, printMessages, "set", false, false);
-    }
-
     /**
      * Merges VariantContexts into a single hybrid.  Takes genotypes for common samples in priority order, if provided.
      * If uniqifySamples is true, the priority order is ignored and names are created by concatenating the VC name with
@@ -486,12 +473,18 @@ public class VariantContextUtils {
      * @param setKey                    the key name of the set
      * @param filteredAreUncalled       are filtered records uncalled?
      * @param mergeInfoWithMaxAC        should we merge in info from the VC with maximum allele count?
-     * @return new VariantContext
+     * @return new VariantContext       representing the merge of unsortedVCs
      */
-    public static VariantContext simpleMerge(GenomeLocParser genomeLocParser, Collection unsortedVCs, List priorityListOfVCs,
-                                             FilteredRecordMergeType filteredRecordMergeType, GenotypeMergeType genotypeMergeOptions,
-                                             boolean annotateOrigin, boolean printMessages, String setKey,
-                                             boolean filteredAreUncalled, boolean mergeInfoWithMaxAC ) {
+    public static VariantContext simpleMerge(final GenomeLocParser genomeLocParser,
+                                             final Collection unsortedVCs,
+                                             final List priorityListOfVCs,
+                                             final FilteredRecordMergeType filteredRecordMergeType,
+                                             final GenotypeMergeType genotypeMergeOptions,
+                                             final boolean annotateOrigin,
+                                             final boolean printMessages,
+                                             final String setKey,
+                                             final boolean filteredAreUncalled,
+                                             final boolean mergeInfoWithMaxAC ) {
         if ( unsortedVCs == null || unsortedVCs.size() == 0 )
             return null;
 
@@ -514,26 +507,28 @@ public class VariantContextUtils {
             return null;
 
         // establish the baseline info from the first VC
-        VariantContext first = VCs.get(0);
-        String name = first.getSource();
-        GenomeLoc loc = getLocation(genomeLocParser,first);
+        final VariantContext first = VCs.get(0);
+        final String name = first.getSource();
+        final Allele refAllele = determineReferenceAllele(VCs);
 
-        Set alleles = new TreeSet();
-        Map genotypes = new TreeMap();
-        double negLog10PError = -1;
-        Set filters = new TreeSet();
-        Map attributes = new TreeMap();
-        Set inconsistentAttributes = new HashSet();
-        String rsID = null;
+        final Set alleles = new TreeSet();
+        final Set filters = new TreeSet();
+        final Map attributes = new TreeMap();
+        final Set inconsistentAttributes = new HashSet();
+        final Set variantSources = new HashSet(); // contains the set of sources we found in our set of VCs that are variant
+        final Set rsIDs = new LinkedHashSet(1); // most of the time there's one id
+
+        GenomeLoc loc = getLocation(genomeLocParser,first);
         int depth = 0;
         int maxAC = -1;
-        Map attributesWithMaxAC = new TreeMap();
+        final Map attributesWithMaxAC = new TreeMap();
+        double negLog10PError = -1;
         VariantContext vcWithMaxAC = null;
+        Map genotypes = new TreeMap();
 
         // counting the number of filtered and variant VCs
-        int nFiltered = 0, nVariant = 0;
+        int nFiltered = 0;
 
-        Allele refAllele = determineReferenceAllele(VCs);
         boolean remapped = false;
 
         // cycle through and add info from the other VCs, making sure the loc/reference matches
@@ -546,7 +541,7 @@ public class VariantContextUtils {
                 loc = getLocation(genomeLocParser,vc); // get the longest location
 
             nFiltered += vc.isFiltered() ? 1 : 0;
-            nVariant += vc.isVariant() ? 1 : 0;
+            if ( vc.isVariant() ) variantSources.add(vc.getSource());
 
             AlleleMapper alleleMapping = resolveIncompatibleAlleles(refAllele, vc, alleles);
             remapped = remapped || alleleMapping.needsRemapping();
@@ -565,11 +560,10 @@ public class VariantContextUtils {
             // special case DP (add it up) and ID (just preserve it)
             //
             if (vc.hasAttribute(VCFConstants.DEPTH_KEY))
-                depth += Integer.valueOf(vc.getAttributeAsString(VCFConstants.DEPTH_KEY));
-            if (rsID == null && vc.hasID())
-                rsID = vc.getID();
+                depth += vc.getAttributeAsInt(VCFConstants.DEPTH_KEY, 0);
+            if ( vc.hasID() && ! vc.getID().equals(VCFConstants.EMPTY_ID_FIELD) ) rsIDs.add(vc.getID());
             if (mergeInfoWithMaxAC && vc.hasAttribute(VCFConstants.ALLELE_COUNT_KEY)) {
-                String rawAlleleCounts = vc.getAttributeAsString(VCFConstants.ALLELE_COUNT_KEY);
+                String rawAlleleCounts = vc.getAttributeAsString(VCFConstants.ALLELE_COUNT_KEY, null);
                 // lets see if the string contains a , separator
                 if (rawAlleleCounts.contains(VCFConstants.INFO_FIELD_ARRAY_SEPARATOR)) {
                     List alleleCountArray = Arrays.asList(rawAlleleCounts.substring(1, rawAlleleCounts.length() - 1).split(VCFConstants.INFO_FIELD_ARRAY_SEPARATOR));
@@ -612,7 +606,9 @@ public class VariantContextUtils {
 
         // if we have more alternate alleles in the merged VC than in one or more of the original VCs, we need to strip out the GL/PLs (because they are no longer accurate)
         for ( VariantContext vc : VCs ) {
-            if ( vc.alleles.size() != alleles.size() ) {
+            if (vc.alleles.size() == 1)
+                continue;
+            if ( vc.alleles.size() != alleles.size()) {
                 genotypes = stripPLs(genotypes);
                 break;
             }
@@ -627,20 +623,20 @@ public class VariantContextUtils {
         if ( filteredRecordMergeType == FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED && nFiltered != VCs.size() )
             filters.clear();
 
-        // we care about where the call came from
-        if ( annotateOrigin ) {
+
+        if ( annotateOrigin ) { // we care about where the call came from
             String setValue;
-            if ( nFiltered == 0 && nVariant == priorityListOfVCs.size() )                   // nothing was unfiltered
-                setValue = "Intersection";
+            if ( nFiltered == 0 && variantSources.size() == priorityListOfVCs.size() ) // nothing was unfiltered
+                setValue = MERGE_INTERSECTION;
             else if ( nFiltered == VCs.size() )     // everything was filtered out
-                setValue = "FilteredInAll";
-            else if ( nVariant == 0 )               // everyone was reference
-                setValue = "ReferenceInAll";
-            else {                                  // we are filtered in some subset
-                List s = new ArrayList();
+                setValue = MERGE_FILTER_IN_ALL;
+            else if ( variantSources.isEmpty() )               // everyone was reference
+                setValue = MERGE_REF_IN_ALL;
+            else {
+                LinkedHashSet s = new LinkedHashSet();
                 for ( VariantContext vc : VCs )
                     if ( vc.isVariant() )
-                        s.add( vc.isFiltered() ? "filterIn" + vc.getSource() : vc.getSource() );
+                        s.add( vc.isFiltered() ? MERGE_FILTER_PREFIX + vc.getSource() : vc.getSource() );
                 setValue = Utils.join("-", s);
             }
 
@@ -652,8 +648,10 @@ public class VariantContextUtils {
 
         if ( depth > 0 )
             attributes.put(VCFConstants.DEPTH_KEY, String.valueOf(depth));
-        if ( rsID != null )
-            attributes.put(VariantContext.ID_KEY, rsID);
+
+        if ( ! rsIDs.isEmpty() ) {
+            attributes.put(VariantContext.ID_KEY, Utils.join(",", rsIDs));
+        }
 
         VariantContext merged = new VariantContext(name, loc.getContig(), loc.getStart(), loc.getStop(), alleles, genotypes, negLog10PError, filters, (mergeInfoWithMaxAC ? attributesWithMaxAC : attributes) );
         // Trim the padded bases of all alleles if necessary
@@ -663,6 +661,18 @@ public class VariantContextUtils {
         return merged;
     }
 
+    public static boolean allelesAreSubset(VariantContext vc1, VariantContext vc2) {
+        // if all alleles of vc1 are a contained in alleles of vc2, return true
+        if (!vc1.getReference().equals(vc2.getReference()))
+            return false;
+
+        for (Allele a :vc1.getAlternateAlleles()) {
+            if (!vc2.getAlternateAlleles().contains(a))
+                return false;
+        }
+
+        return true;
+    }
     public static VariantContext createVariantContextWithTrimmedAlleles(VariantContext inputVC) {
         // see if we need to trim common reference base from all alleles
         boolean trimVC;
@@ -739,7 +749,7 @@ public class VariantContextUtils {
         Map newGs = new HashMap(genotypes.size());
 
         for ( Map.Entry g : genotypes.entrySet() ) {
-            newGs.put(g.getKey(), g.getValue().hasLikelihoods() ? Genotype.removePLs(g.getValue()) : g.getValue());
+            newGs.put(g.getKey(), g.getValue().hasLikelihoods() ? removePLs(g.getValue()) : g.getValue());
         }
 
         return newGs;
@@ -748,9 +758,46 @@ public class VariantContextUtils {
     public static Map> separateVariantContextsByType(Collection VCs) {
         HashMap> mappedVCs = new HashMap>();
         for ( VariantContext vc : VCs ) {
-            if ( !mappedVCs.containsKey(vc.getType()) )
-                mappedVCs.put(vc.getType(), new ArrayList());
-            mappedVCs.get(vc.getType()).add(vc);
+
+            // look at previous variant contexts of different type. If:
+            // a) otherVC has alleles which are subset of vc, remove otherVC from its list and add otherVC to  vc's list
+            // b) vc has alleles which are subset of otherVC. Then, add vc to otherVC's type list (rather, do nothing since vc will be added automatically to its list)
+            // c) neither: do nothing, just add vc to its own list
+            boolean addtoOwnList = true;
+            for (VariantContext.Type type : VariantContext.Type.values()) {
+                if (type.equals(vc.getType()))
+                    continue;
+
+                if (!mappedVCs.containsKey(type))
+                    continue;
+
+                List vcList = mappedVCs.get(type);
+                for (int k=0; k <  vcList.size(); k++) {
+                    VariantContext otherVC = vcList.get(k);
+                    if (allelesAreSubset(otherVC,vc)) {
+                        // otherVC has a type different than vc and its alleles are a subset of vc: remove otherVC from its list and add it to vc's type list
+                        vcList.remove(k);
+                        // avoid having empty lists
+                        if (vcList.size() == 0)
+                            mappedVCs.remove(vcList);
+                        if ( !mappedVCs.containsKey(vc.getType()) )
+                            mappedVCs.put(vc.getType(), new ArrayList());
+                        mappedVCs.get(vc.getType()).add(otherVC);
+                        break;
+                    }
+                    else if (allelesAreSubset(vc,otherVC)) {
+                        // vc has a type different than otherVC and its alleles are a subset of VC: add vc to otherVC's type list and don't add to its own
+                        mappedVCs.get(type).add(vc);
+                        addtoOwnList = false;
+                        break;
+                    }
+                }
+            }
+            if (addtoOwnList) {
+                if ( !mappedVCs.containsKey(vc.getType()) )
+                    mappedVCs.put(vc.getType(), new ArrayList());
+                mappedVCs.get(vc.getType()).add(vc);
+                }
         }
 
         return mappedVCs;
@@ -1147,9 +1194,7 @@ public class VariantContextUtils {
         for (String orAttrib : MERGE_OR_ATTRIBS) {
             boolean attribVal = false;
             for (VariantContext vc : vcList) {
-                Boolean val = vc.getAttributeAsBooleanNoException(orAttrib);
-                if (val != null)
-                    attribVal = (attribVal || val);
+                attribVal = vc.getAttributeAsBoolean(orAttrib, false);
                 if (attribVal) // already true, so no reason to continue:
                     break;
             }
@@ -1159,7 +1204,7 @@ public class VariantContextUtils {
         // Merge ID fields:
         String iDVal = null;
         for (VariantContext vc : vcList) {
-            String val = vc.getAttributeAsStringNoException(VariantContext.ID_KEY);
+            String val = vc.getAttributeAsString(VariantContext.ID_KEY, null);
             if (val != null && !val.equals(VCFConstants.EMPTY_ID_FIELD)) {
                 if (iDVal == null)
                     iDVal = val;
@@ -1239,8 +1284,10 @@ public class VariantContextUtils {
 
         public PhaseAndQuality(Genotype gt) {
             this.isPhased = gt.isPhased();
-            if (this.isPhased)
-                this.PQ = gt.getAttributeAsDoubleNoException(ReadBackedPhasingWalker.PQ_KEY);
+            if (this.isPhased) {
+                this.PQ = gt.getAttributeAsDouble(ReadBackedPhasingWalker.PQ_KEY, -1);
+                if ( this.PQ == -1 ) this.PQ = null;
+            }
         }
     }
 
diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java
index 63faf1ab9..35a81770d 100755
--- a/public/java/test/org/broadinstitute/sting/BaseTest.java
+++ b/public/java/test/org/broadinstitute/sting/BaseTest.java
@@ -132,15 +132,21 @@ public abstract class BaseTest {
      */
     public static class TestDataProvider {
         private static final Map> tests = new HashMap>();
+        private final String name;
 
         /**
          * Create a new TestDataProvider instance bound to the class variable C
          * @param c
          */
-        public TestDataProvider(Class c) {
+        public TestDataProvider(Class c, String name) {
             if ( ! tests.containsKey(c) )
                 tests.put(c, new ArrayList());
             tests.get(c).add(this);
+            this.name = name;
+        }
+
+        public TestDataProvider(Class c) {
+            this(c, "");
         }
 
         /**
@@ -153,6 +159,11 @@ public abstract class BaseTest {
             for ( Object x : tests.get(c) ) params2.add(new Object[]{x});
             return params2.toArray(new Object[][]{});
         }
+
+        @Override
+        public String toString() {
+            return "TestDataProvider("+name+")";
+        }
     }
 
     /**
diff --git a/public/java/test/org/broadinstitute/sting/WalkerTest.java b/public/java/test/org/broadinstitute/sting/WalkerTest.java
index 386c17659..a1817e3c7 100755
--- a/public/java/test/org/broadinstitute/sting/WalkerTest.java
+++ b/public/java/test/org/broadinstitute/sting/WalkerTest.java
@@ -75,7 +75,7 @@ public class WalkerTest extends BaseTest {
         Index indexFromOutputFile = IndexFactory.createIndex(resultFile, new VCFCodec());
         Index dynamicIndex = IndexFactory.loadIndex(indexFile.getAbsolutePath());
 
-        if ( ! indexFromOutputFile.equalsIgnoreTimestamp(dynamicIndex) ) {
+        if ( ! indexFromOutputFile.equalsIgnoreProperties(dynamicIndex) ) {
             Assert.fail(String.format("Index on disk from indexing on the fly not equal to the index created after the run completed.  FileIndex %s vs. on-the-fly %s%n",
                     indexFromOutputFile.getProperties(),
                     dynamicIndex.getProperties()));
diff --git a/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/FeatureManagerUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/FeatureManagerUnitTest.java
index bae8e99ed..e8799e2ab 100644
--- a/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/FeatureManagerUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/FeatureManagerUnitTest.java
@@ -56,6 +56,7 @@ public class FeatureManagerUnitTest extends BaseTest {
     private static final File VCF3_FILE = new File(validationDataLocation + "vcfexample3.vcf");
     private static final File VCF4_FILE = new File(testDir + "HiSeq.10000.vcf");
     private static final File VCF4_FILE_GZ = new File(testDir + "HiSeq.10000.vcf.gz");
+    private static final File VCF4_FILE_BGZIP = new File(testDir + "HiSeq.10000.bgzip.vcf.gz");
 
     private FeatureManager manager;
     private GenomeLocParser genomeLocParser;
@@ -109,6 +110,7 @@ public class FeatureManagerUnitTest extends BaseTest {
         new FMTest(VariantContext.class, VCF3Codec.class, "VCF3", VCF3_FILE);
         new FMTest(VariantContext.class, VCFCodec.class, "VCF", VCF4_FILE);
         new FMTest(VariantContext.class, VCFCodec.class, "VCF", VCF4_FILE_GZ);
+        new FMTest(VariantContext.class, VCFCodec.class, "VCF", VCF4_FILE_BGZIP);
         new FMTest(TableFeature.class, BedTableCodec.class, "bedtable", null);
         return FMTest.getTests(FMTest.class);
     }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilderUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilderUnitTest.java
index ae218e898..724c343e4 100644
--- a/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilderUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrackBuilderUnitTest.java
@@ -29,7 +29,6 @@ import net.sf.picard.reference.IndexedFastaSequenceFile;
 import net.sf.samtools.SAMSequenceDictionary;
 import org.broad.tribble.Tribble;
 import org.broad.tribble.index.Index;
-import org.broadinstitute.sting.gatk.refdata.tracks.RMDTrackBuilder;
 import org.broadinstitute.sting.utils.codecs.vcf.VCF3Codec;
 import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
 import org.broadinstitute.sting.utils.exceptions.UserException;
@@ -45,7 +44,6 @@ import org.testng.annotations.Test;
 
 import java.io.*;
 import java.nio.channels.FileChannel;
-import java.util.Map;
 
 
 /**
@@ -164,7 +162,7 @@ public class RMDTrackBuilderUnitTest extends BaseTest {
         try {
             Index idx = builder.loadIndex(vcfFile, new VCFCodec());
             // catch any exception; this call should pass correctly
-            SAMSequenceDictionary dict =  RMDTrackBuilder.getSequenceDictionaryFromProperties(idx);
+            SAMSequenceDictionary dict =  IndexDictionaryUtils.getSequenceDictionaryFromProperties(idx);
         } catch (IOException e) {
             e.printStackTrace();
             Assert.fail("IO exception unexpected" + e.getMessage());
diff --git a/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
index c0d32a05b..7f4d96add 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/traversals/TraverseReadsUnitTest.java
@@ -127,6 +127,7 @@ public class TraverseReadsUnitTest extends BaseTest {
         Object accumulator = countReadWalker.reduceInit();
 
         while (shardStrategy.hasNext()) {
+            traversalEngine.startTimersIfNecessary();
             Shard shard = shardStrategy.next();
 
             if (shard == null) {
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/CNV/SymbolicAllelesIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/CNV/SymbolicAllelesIntegrationTest.java
index b4a8498e1..1b2a6e82e 100644
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/CNV/SymbolicAllelesIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/CNV/SymbolicAllelesIntegrationTest.java
@@ -33,7 +33,7 @@ public class SymbolicAllelesIntegrationTest extends WalkerTest {
         WalkerTestSpec spec = new WalkerTestSpec(
                 baseTestString(b36KGReference, "symbolic_alleles_2.vcf"),
                 1,
-                Arrays.asList("6645babc8c7d46be0da223477c7b1291"));
+                Arrays.asList("3008d6f5044bc14801e5c58d985dec72"));
         executeTest("Test symbolic alleles mixed in with non-symbolic alleles", spec);
     }
 }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/SnpEffUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/SnpEffUnitTest.java
new file mode 100644
index 000000000..462abeba1
--- /dev/null
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/SnpEffUnitTest.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.gatk.walkers.annotator;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+import org.broadinstitute.sting.gatk.walkers.annotator.SnpEff.SnpEffEffect;
+
+public class SnpEffUnitTest {
+
+    @Test
+    public void testParseWellFormedEffect() {
+        String effectName = "NON_SYNONYMOUS_CODING";
+        String[] effectMetadata = { "MODERATE", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990", "exon_1_69037_69829" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertTrue( effect.isWellFormed() && effect.isCoding() );
+    }
+
+    @Test
+    public void testParseInvalidEffectNameEffect() {
+        String effectName = "MADE_UP_EFFECT";
+        String[] effectMetadata = { "MODERATE", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990", "exon_1_69037_69829" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertFalse(effect.isWellFormed());
+    }
+
+    @Test
+    public void testParseInvalidEffectImpactEffect() {
+        String effectName = "NON_SYNONYMOUS_CODING";
+        String[] effectMetadata = { "MEDIUM", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990", "exon_1_69037_69829" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertFalse(effect.isWellFormed());
+    }
+
+    @Test
+    public void testParseWrongNumberOfMetadataFieldsEffect() {
+        String effectName = "NON_SYNONYMOUS_CODING";
+        String[] effectMetadata = { "MODERATE", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertFalse(effect.isWellFormed());
+    }
+
+    @Test
+    public void testParseSnpEffWarningEffect() {
+        String effectName = "NON_SYNONYMOUS_CODING";
+        String[] effectMetadata = { "MODERATE", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990", "exon_1_69037_69829", "SNPEFF_WARNING" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertTrue( ! effect.isWellFormed() && effect.getParseError().equals("SnpEff issued the following warning: SNPEFF_WARNING") );
+    }
+
+    @Test
+    public void testParseSnpEffErrorEffect() {
+        String effectName = "NON_SYNONYMOUS_CODING";
+        String[] effectMetadata = { "MODERATE", "Aca/Gca", "T/A", "OR4F5", "protein_coding", "CODING", "ENST00000534990", "exon_1_69037_69829", "", "SNPEFF_ERROR" };
+
+        SnpEffEffect effect = new SnpEffEffect(effectName, effectMetadata);
+        Assert.assertTrue( ! effect.isWellFormed() && effect.getParseError().equals("SnpEff issued the following error: SNPEFF_ERROR") );
+    }
+}
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java
index 832079807..04bff8d41 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/annotator/VariantAnnotatorIntegrationTest.java
@@ -1,6 +1,7 @@
 package org.broadinstitute.sting.gatk.walkers.annotator;
 
 import org.broadinstitute.sting.WalkerTest;
+import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.testng.annotations.Test;
 
 import java.util.Arrays;
@@ -129,12 +130,24 @@ public class VariantAnnotatorIntegrationTest extends WalkerTest {
     @Test
     public void testSnpEffAnnotations() {
         WalkerTestSpec spec = new WalkerTestSpec(
-            "-T VariantAnnotator -R " + b37KGReference + " -NO_HEADER -o %s -A SnpEff --variant " +
-            validationDataLocation + "1000G.exomes.vcf --snpEffFile  " + validationDataLocation +
-            "snpEff_1.9.6_1000G.exomes.vcf_hg37.61.out -L 1:26,000,000-26,500,000",
+            "-T VariantAnnotator -R " + hg19Reference + " -NO_HEADER -o %s -A SnpEff --variant " +
+            validationDataLocation + "1kg_exomes_unfiltered.AFR.unfiltered.vcf --snpEffFile  " + validationDataLocation +
+            "snpEff.AFR.unfiltered.vcf -L 1:1-1,500,000 -L 2:232,325,429",
             1,
-            Arrays.asList("03eae1dab19a9358250890594bf53607")
+            Arrays.asList("122321a85e448f21679f6ca15c5e22ad")
         );
         executeTest("Testing SnpEff annotations", spec);
     }
+
+    @Test
+    public void testSnpEffAnnotationsUnsupportedVersion() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+            "-T VariantAnnotator -R " + hg19Reference + " -NO_HEADER -o %s -A SnpEff --variant " +
+            validationDataLocation + "1kg_exomes_unfiltered.AFR.unfiltered.vcf --snpEffFile  " + validationDataLocation +
+            "snpEff.AFR.unfiltered.unsupported.version.vcf -L 1:1-1,500,000",
+            1,
+            UserException.class
+        );
+        executeTest("Testing SnpEff annotations (unsupported version)", spec);
+    }
 }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java
index 5f759fdbf..1a01ef8e8 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/beagle/BeagleIntegrationTest.java
@@ -41,7 +41,7 @@ public class BeagleIntegrationTest extends WalkerTest {
                         "--beagleR2:BEAGLE " + beagleValidationDataLocation + "inttestbgl.r2 " +
                         "--beagleProbs:BEAGLE " + beagleValidationDataLocation + "inttestbgl.gprobs " +
                         "--beaglePhased:BEAGLE " + beagleValidationDataLocation + "inttestbgl.phased " +
-                        "-o %s -NO_HEADER", 1, Arrays.asList("3531451e84208264104040993889aaf4"));
+                        "-o %s -NO_HEADER", 1, Arrays.asList("b445d280fd8fee1eeb4aacb3f5a54847"));
         executeTest("test BeagleOutputToVCF", spec);
     }
    
@@ -72,7 +72,7 @@ public class BeagleIntegrationTest extends WalkerTest {
                 "--beagleR2:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.r2 "+
                 "--beagleProbs:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.gprobs.bgl "+
                 "--beaglePhased:beagle /humgen/gsa-hpprojects/GATK/data/Validation_Data/EUR_beagle_in_test.phased.bgl "+
-                "-L 20:1-70000 -o %s -NO_HEADER ",1,Arrays.asList("8dd6ec53994fb46c5c22af8535d22965"));
+                "-L 20:1-70000 -o %s -NO_HEADER ",1,Arrays.asList("51a57ea565176edd96d907906914b0ee"));
 
         executeTest("testBeagleChangesSitesToRef",spec);
     }
diff --git a/public/java/test/org/broadinstitute/sting/utils/genotype/DiploidGenotypeUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotypeUnitTest.java
similarity index 95%
rename from public/java/test/org/broadinstitute/sting/utils/genotype/DiploidGenotypeUnitTest.java
rename to public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotypeUnitTest.java
index e4f8b12e3..4e72b37a4 100644
--- a/public/java/test/org/broadinstitute/sting/utils/genotype/DiploidGenotypeUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/DiploidGenotypeUnitTest.java
@@ -1,5 +1,6 @@
-package org.broadinstitute.sting.utils.genotype;
+package org.broadinstitute.sting.gatk.walkers.genotyper;
 
+import org.broadinstitute.sting.gatk.walkers.genotyper.DiploidGenotype;
 import org.testng.Assert;
 import org.broadinstitute.sting.BaseTest;
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsUnitTest.java
index 9882ce869..425b969e2 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/GenotypeLikelihoodsUnitTest.java
@@ -1,7 +1,6 @@
 package org.broadinstitute.sting.gatk.walkers.genotyper;
 
 import org.testng.Assert;
-import org.broadinstitute.sting.utils.genotype.DiploidGenotype;
 import org.broadinstitute.sting.BaseTest;
 import org.testng.annotations.Test;
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
index da0c8f81f..488b3ccd9 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/genotyper/UnifiedGenotyperIntegrationTest.java
@@ -18,6 +18,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
     private final static String baseCommand = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm BOTH --dbsnp " + b36dbSNP129;
     private final static String baseCommandIndels = "-T UnifiedGenotyper -R " + b36KGReference + " -NO_HEADER -glm INDEL --dbsnp " + b36dbSNP129;
+    private final static String baseCommandIndelsb37 = "-T UnifiedGenotyper -R " + b37KGReference + " -NO_HEADER -glm INDEL --dbsnp " + b37dbSNP132;
 
     // --------------------------------------------------------------------------------------------------------------
     //
@@ -28,28 +29,10 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testMultiSamplePilot1() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "low_coverage_CEU.chr1.10k-11k.bam -o %s -L 1:10,022,000-10,025,000", 1,
-                Arrays.asList("149e6ad9b3fd23551254a691286a96b3"));
+                Arrays.asList("e6639ea2dc81635c706e6c35921406d7"));
         executeTest("test MultiSample Pilot1", spec);
     }
 
-    //    @Test
-    // todo - currently not working because when calling indels, using GENOTYPE_GIVEN_ALLELES yields a different result than in normal mode. To be fixed when extended events are removed.
-    public void testMultiSamplePilot2AndRecallingWithAlleles() {
-        String md5 = "b45636b29891f9df573ad2af6f507ee0";
-
-        WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,050,000", 1,
-                Arrays.asList(md5));
-        List result = executeTest("test MultiSample Pilot2", spec1).getFirst();
-
-        GenomeAnalysisEngine.resetRandomGenerator();
-
-        WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
-                baseCommand + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + result.get(0).getAbsolutePath() + " -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,050,000", 1,
-                Arrays.asList(md5));
-        executeTest("test MultiSample Pilot2 with alleles passed in", spec2);       
-    }
-
     @Test
     public void testWithAllelesPassedIn() {
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
@@ -59,7 +42,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         WalkerTest.WalkerTestSpec spec2 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " --output_mode EMIT_ALL_SITES --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "allelesForUG.vcf -I " + validationDataLocation + "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,025,000", 1,
-                Arrays.asList("ec43daadfb15b00b41aeb0017a45df0b"));
+                Arrays.asList("6458f3b8fe4954e2ffc2af972aaab19e"));
         executeTest("test MultiSample Pilot2 with alleles passed in and emitting all sites", spec2);
     }
 
@@ -67,7 +50,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     public void testSingleSamplePilot2() {
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000", 1,
-                Arrays.asList("82d469145c174486ccc494884852cc58"));
+                Arrays.asList("d1cbd1fb9f3f7323941a95bc2def7e5a"));
         executeTest("test SingleSample Pilot2", spec);
     }
 
@@ -77,7 +60,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     //
     // --------------------------------------------------------------------------------------------------------------
 
-    private final static String COMPRESSED_OUTPUT_MD5 = "a5a9f38c645d6004d4640765a8b77ce4";
+    private final static String COMPRESSED_OUTPUT_MD5 = "2732b169cdccb21eb3ea00429619de79";
 
     @Test
     public void testCompressedOutput() {
@@ -87,15 +70,6 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
         executeTest("test compressed output", spec);
     }
 
-    // todo -- fixme
-//    @Test
-//    public void testCompressedOutputParallel() {
-//        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
-//                baseCommand + " -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,100,000 -nt 4", 1,
-//                Arrays.asList("gz"), Arrays.asList(COMPRESSED_OUTPUT_MD5));
-//        executeTest("testCompressedOutput-nt4", spec);
-//    }
-
     // --------------------------------------------------------------------------------------------------------------
     //
     // testing parallelization
@@ -107,7 +81,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
 
         // Note that we need to turn off any randomization for this to work, so no downsampling and no annotations
 
-        String md5 = "0a45761c0e557d9c2080eb9e7f4f6c41";
+        String md5 = "cbac3960bbcb9d6192c57549208c182c";
 
         WalkerTest.WalkerTestSpec spec1 = new WalkerTest.WalkerTestSpec(
                 baseCommand + " -dt NONE -G none -I " + validationDataLocation + "NA12878.1kg.p2.chr1_10mb_11_mb.SLX.bam -o %s -L 1:10,000,000-10,075,000", 1,
@@ -186,8 +160,8 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
     @Test
     public void testHeterozyosity() {
         HashMap e = new HashMap();
-        e.put( 0.01, "af5199fbc0853cf5888acdcc88f012bc" );
-        e.put( 1.0 / 1850, "4e6938645ccde1fdf204ffbf4e88170f" );
+        e.put( 0.01, "aed69402ddffe7f2ed5ca98563bfba02" );
+        e.put( 1.0 / 1850, "fa94a059f08c1821b721335d93ed2ea5" );
 
         for ( Map.Entry entry : e.entrySet() ) {
             WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
@@ -211,7 +185,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("213ebaaaacf850312d885e918eb33500"));
+                Arrays.asList("1c080e6596d4c830bb5d147b04e2a82c"));
 
         executeTest(String.format("test multiple technologies"), spec);
     }
@@ -230,7 +204,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -L 1:10,000,000-10,100,000" +
                         " -baq CALCULATE_AS_NECESSARY",
                 1,
-                Arrays.asList("3aecba34a89f3525afa57a38dc20e6cd"));
+                Arrays.asList("9129ad748ca3be2d3b321d2d7e83ae5b"));
 
         executeTest(String.format("test calling with BAQ"), spec);
     }
@@ -249,7 +223,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -o %s" +
                         " -L 1:10,000,000-10,500,000",
                 1,
-                Arrays.asList("043973c719a85de29a35a33a674616fb"));
+                Arrays.asList("0bece77ce6bc447438ef9b2921b2dc41"));
 
         executeTest(String.format("test indel caller in SLX"), spec);
     }
@@ -264,7 +238,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         " -minIndelCnt 1" +
                         " -L 1:10,000,000-10,100,000",
                 1,
-                Arrays.asList("68d4e6c1849e892467aed61c33e7bf24"));
+                Arrays.asList("5fe98ee853586dc9db58f0bc97daea63"));
 
         executeTest(String.format("test indel caller in SLX witn low min allele count"), spec);
     }
@@ -277,7 +251,7 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                          " -o %s" +
                          " -L 1:10,000,000-10,500,000",
                  1,
-                 Arrays.asList("f86d453c5d2d2f33fb28ae2050658a5e"));
+                 Arrays.asList("790b1a1d6ab79eee8c24812bb8ca6fae"));
 
          executeTest(String.format("test indel calling, multiple technologies"), spec);
      }
@@ -296,7 +270,21 @@ public class UnifiedGenotyperIntegrationTest extends WalkerTest {
                         "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,100,000", 1,
                 Arrays.asList("94977d6e42e764280e9deaf4e3ac8c80"));
         executeTest("test MultiSample Pilot2 indels with alleles passed in and emitting all sites", spec2);
+
+        WalkerTest.WalkerTestSpec spec3 = new WalkerTest.WalkerTestSpec(
+                baseCommandIndels + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2.20101123.indels.sites.vcf -I " + validationDataLocation +
+                        "pilot2_daughters.chr20.10k-11k.bam -o %s -L 20:10,000,000-10,080,000", 1,
+                Arrays.asList("e66b7321e2ac91742ad3ef91040daafd"));
+        executeTest("test MultiSample Pilot2 indels with complicated records", spec3);
+
+        WalkerTest.WalkerTestSpec spec4 = new WalkerTest.WalkerTestSpec(
+                baseCommandIndelsb37 + " --genotyping_mode GENOTYPE_GIVEN_ALLELES -alleles " + validationDataLocation + "ALL.wgs.union_v2_chr20_100_110K.20101123.indels.sites.vcf -I " + validationDataLocation +
+                        "phase1_GBR_realigned.chr20.100K-110K.bam -o %s -L 20:100,000-110,000", 1,
+                Arrays.asList("4be308fd9e8167ebee677f62a7a753b7"));
+        executeTest("test MultiSample 1000G Phase1 indels with complicated records emitting all sites", spec4);
+
     }
 
 
+
 }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
index 3503a2353..9fe253ecb 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java
@@ -6,7 +6,7 @@ import org.testng.annotations.Test;
 import java.util.Arrays;
 
 public class VariantEvalIntegrationTest extends WalkerTest {
-    private static String variantEvalTestDataRoot = validationDataLocation + "/VariantEval";
+    private static String variantEvalTestDataRoot = validationDataLocation + "VariantEval";
     private static String fundamentalTestVCF = variantEvalTestDataRoot + "/" + "FundamentalsTest.annotated.db.subset.snps_and_indels.vcf";
     private static String fundamentalTestSNPsVCF = variantEvalTestDataRoot + "/" + "FundamentalsTest.annotated.db.subset.final.vcf";
     private static String fundamentalTestSNPsOneSampleVCF = variantEvalTestDataRoot + "/" + "FundamentalsTest.annotated.db.subset.final.HG00625.vcf";
@@ -14,6 +14,47 @@ public class VariantEvalIntegrationTest extends WalkerTest {
     private static String cmdRoot = "-T VariantEval" +
             " -R " + b36KGReference;
 
+    @Test
+    public void testFunctionClassWithSnpeff() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                                buildCommandLine(
+                                        "-T VariantEval",
+                                        "-R " + b37KGReference,
+                                        "--dbsnp " + b37dbSNP132,
+                                        "--eval " + validationDataLocation + "snpEff.AFR.unfiltered.VariantAnnotator.output.vcf",
+                                        "-noEV",
+                                        "-EV TiTvVariantEvaluator",
+                                        "-noST",
+                                        "-ST FunctionalClass",
+                                        "-BTI eval",
+                                        "-o %s"
+                                ),
+                                1,
+                                Arrays.asList("d9dcb352c53106f54fcc981f15d35a90")
+                              );
+        executeTest("testFunctionClassWithSnpeff", spec);
+    }
+
+    @Test
+    public void testStratifySamplesAndExcludeMonomorphicSites() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+                                buildCommandLine(
+                                        "-T VariantEval",
+                                        "-R " + b37KGReference,
+                                        "--dbsnp " + b37dbSNP132,
+                                        "--eval " + variantEvalTestDataRoot + "/CEU.trio.callsForVE.vcf",
+                                        "-noEV",
+                                        "-EV TiTvVariantEvaluator",
+                                        "-ST Sample",
+                                        "-BTI eval",
+                                        "-o %s"
+                                ),
+                                1,
+                                Arrays.asList("6a71b17c19f5914c277a99f45f5d9c39")
+                              );
+        executeTest("testStratifySamplesAndExcludeMonomorphicSites", spec);
+    }
+
     @Test
     public void testFundamentalsCountVariantsSNPsAndIndels() {
         WalkerTestSpec spec = new WalkerTestSpec(
@@ -236,7 +277,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
                 " --eval " + validationDataLocation + "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.vcf" +
                 " --comp:comp_genotypes,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.head.vcf";
         WalkerTestSpec spec = new WalkerTestSpec(withSelect(tests, "DP < 50", "DP50") + " " + extraArgs + " -ST CpG -o %s",
-                1, Arrays.asList("2df4f8911ffc3c8d042298723ed465f8"));
+                1, Arrays.asList("f70997b6a3e7fdc89d11e1d61a2463d4"));
         executeTestParallel("testSelect1", spec);
     }
 
@@ -253,7 +294,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
     @Test
     public void testCompVsEvalAC() {
         String extraArgs = "-T VariantEval -R "+b36KGReference+" -o %s -ST CpG -EV GenotypeConcordance --eval:evalYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.very.few.lines.vcf --comp:compYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.fake.genotypes.ac.test.vcf";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("ed54aa127b173d8ad8b6482f2a929a42"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("407682de41dcf139ea635e9cda21b912"));
         executeTestParallel("testCompVsEvalAC",spec);
     }
 
@@ -261,17 +302,17 @@ public class VariantEvalIntegrationTest extends WalkerTest {
         return String.format("%s -select '%s' -selectName %s", cmd, select, name);
     }
 
-    @Test
+    @Test(enabled = false) // no longer supported in the GATK
     public void testTranches() {
         String extraArgs = "-T VariantEval -R "+ hg18Reference +" --eval " + validationDataLocation + "GA2.WEx.cleaned.ug.snpfiltered.indelfiltered.optimized.vcf -o %s -EV TiTvVariantEvaluator -L chr1 -noEV -ST CpG -tf " + testDir + "tranches.6.txt";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("984df6e94a546294fc7e0846cbac2dfe"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("6af2b9959aa1778a5b712536de453952"));
         executeTestParallel("testTranches",spec);
     }
 
     @Test
     public void testCompOverlap() {
         String extraArgs = "-T VariantEval -R " + b37KGReference + " -L " + validationDataLocation + "VariantEval/pacbio.hg19.intervals --comp:comphapmap " + comparisonDataLocation + "Validated/HapMap/3.3/genotypes_r27_nr.b37_fwd.vcf --eval " + validationDataLocation + "VariantEval/pacbio.ts.recalibrated.vcf -noEV -EV CompOverlap -sn NA12878 -noST -ST Novelty -o %s";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("462d4784dd55294ef9d5118217b157a5"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("009ecc8376a20dce81ff5299ef6bfecb"));
         executeTestParallel("testCompOverlap",spec);
     }
 
@@ -283,7 +324,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
                            " --dbsnp " + b37dbSNP132 +
                            " --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" +
                            " -noST -ST Novelty -o %s";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("18c44636e36d6657110bf984f8eac181"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("424c9d438b1faa59b2c29413ba32f37b"));
         executeTestParallel("testEvalTrackWithoutGenotypes",spec);
     }
 
@@ -295,7 +336,7 @@ public class VariantEvalIntegrationTest extends WalkerTest {
                 " --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" +
                 " --eval:evalBC " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bc.sites.vcf" +
                 " -noST -ST Novelty -o %s";
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("1b8ae4fd10de0888bd843f833859d990"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("18fa0b89ebfff51141975d7e4ce7a159"));
         executeTestParallel("testMultipleEvalTracksWithoutGenotypes",spec);
     }
 
@@ -312,13 +353,13 @@ public class VariantEvalIntegrationTest extends WalkerTest {
                            " -noST -noEV -ST Novelty -EV CompOverlap" +
                            " -o %s";
 
-        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("a3c2177849cb00fdff99574cff7f0e4f"));
+        WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("0b81d97f843ec4a1a4222d1f9949bfca"));
         executeTestParallel("testMultipleCompTracks",spec);
     }
 
     @Test
     public void testPerSampleAndSubsettedSampleHaveSameResults() {
-        String md5 = "dab415cc76846e18fcf8c78f2b2ee033";
+        String md5 = "b0565ac61b2860248e4abd478a177b5e";
 
         WalkerTestSpec spec = new WalkerTestSpec(
                 buildCommandLine(
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
index a5b0412e8..c81891ac6 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java
@@ -41,11 +41,9 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
         //System.out.printf("PARAMS FOR %s is %s%n", vcf, clusterFile);
         WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
                 "-R " + b37KGReference +
-                        " -known:prior=10.0 " + GATKDataLocation + "dbsnp_132_b37.leftAligned.vcf" +
-                        " -training:prior=15.0 " + comparisonDataLocation + "Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf" +
-                        " -truth:prior=15.0 " + comparisonDataLocation + "Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf" +
-                        " -training:prior=12.0 " + comparisonDataLocation + "Validated/Omni2.5_chip/Omni25_sites_1525_samples.b37.vcf" +
-                        " -truth:prior=12.0 " + comparisonDataLocation + "Validated/Omni2.5_chip/Omni25_sites_1525_samples.b37.vcf" +
+                        " -resource:known=true,prior=10.0 " + GATKDataLocation + "dbsnp_132_b37.leftAligned.vcf" +
+                        " -resource:truth=true,training=true,prior=15.0 " + comparisonDataLocation + "Validated/HapMap/3.3/sites_r27_nr.b37_fwd.vcf" +
+                        " -resource:training=true,truth=true,prior=12.0 " + comparisonDataLocation + "Validated/Omni2.5_chip/Omni25_sites_1525_samples.b37.vcf" +
                         " -T VariantRecalibrator" +
                         " -input " + params.inVCF +
                         " -L 20:1,000,000-40,000,000" +
@@ -73,5 +71,52 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
                 Arrays.asList(params.cutVCFMD5));
         executeTest("testApplyRecalibration-"+params.inVCF, spec);
     }
+
+    VRTest indel = new VRTest("combined.phase1.chr20.raw.indels.sites.vcf",
+            "6d7ee4cb651c8b666e4a4523363caaff",  // tranches
+            "4759b111a5aa53975d46e0f22c7983bf",  // recal file
+            "5d7e07d8813db96ba3f3dfe4737f83d1"); // cut VCF
+
+    @DataProvider(name = "VRIndelTest")
+    public Object[][] createData2() {
+        return new Object[][]{ {indel} };
+    }
+
+    @Test(dataProvider = "VRIndelTest")
+    public void testVariantRecalibratorIndel(VRTest params) {
+        //System.out.printf("PARAMS FOR %s is %s%n", vcf, clusterFile);
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                "-R " + b37KGReference +
+                        " -resource:known=true,prior=10.0 " + GATKDataLocation + "dbsnp_132_b37.leftAligned.vcf" +
+                        " -resource:training=true,truth=true,prior=15.0 " + comparisonDataLocation + "Validated/Mills_Devine_Indels_2011/ALL.wgs.indels_mills_devine_hg19_leftAligned_collapsed_double_hit.sites.vcf" +
+                        " -T VariantRecalibrator" +
+                        " -input " + params.inVCF +
+                        " -L 20:1,000,000-40,000,000" +
+                        " -an QD -an ReadPosRankSum -an HaplotypeScore" +
+                        " -percentBad 0.08" +
+                        " -mode INDEL -mG 3" +
+                        " --minNumBadVariants 0" +
+                        " --trustAllPolymorphic" + // for speed
+                        " -recalFile %s" +
+                        " -tranchesFile %s",
+                Arrays.asList(params.recalMD5, params.tranchesMD5));
+        executeTest("testVariantRecalibratorIndel-"+params.inVCF, spec).getFirst();
+    }
+
+    @Test(dataProvider = "VRIndelTest",dependsOnMethods="testVariantRecalibratorIndel")
+    public void testApplyRecalibrationIndel(VRTest params) {
+        WalkerTest.WalkerTestSpec spec = new WalkerTest.WalkerTestSpec(
+                "-R " + b37KGReference +
+                        " -T ApplyRecalibration" +
+                        " -L 20:12,000,000-30,000,000" +
+                        " -mode INDEL" +
+                        " -NO_HEADER" +
+                        " -input " + params.inVCF +
+                        " -o %s" +
+                        " -tranchesFile " + MD5DB.getMD5FilePath(params.tranchesMD5, null) +
+                        " -recalFile " + MD5DB.getMD5FilePath(params.recalMD5, null),
+                Arrays.asList(params.cutVCFMD5));
+        executeTest("testApplyRecalibrationIndel-"+params.inVCF, spec);
+    }
 }
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java
index 3267173a7..3b1a97369 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/CombineVariantsIntegrationTest.java
@@ -89,15 +89,15 @@ public class CombineVariantsIntegrationTest extends WalkerTest {
     @Test public void combineWithPLs() { combinePLs("combine.3.vcf", "combine.4.vcf", "0f873fed02aa99db5b140bcd6282c10a"); }
 
     @Test public void combineTrioCalls() { combine2("CEU.trio.2010_03.genotypes.vcf.gz", "YRI.trio.2010_03.genotypes.vcf.gz", "", "1d5a021387a8a86554db45a29f66140f"); } // official project VCF files in tabix format
-    @Test public void combineTrioCallsMin() { combine2("CEU.trio.2010_03.genotypes.vcf.gz", "YRI.trio.2010_03.genotypes.vcf.gz", " -minimalVCF", "20163d60f18a46496f6da744ab5cc0f9"); } // official project VCF files in tabix format
-    @Test public void combine2Indels() { combine2("CEU.dindel.vcf4.trio.2010_06.indel.genotypes.vcf", "CEU.dindel.vcf4.low_coverage.2010_06.indel.genotypes.vcf", "", "f1cf095c2fe9641b7ca1f8ee2c46fd4a"); }
+    @Test public void combineTrioCallsMin() { combine2("CEU.trio.2010_03.genotypes.vcf.gz", "YRI.trio.2010_03.genotypes.vcf.gz", " -minimalVCF", "96941ee177b0614a9879af0ac3218963"); } // official project VCF files in tabix format
+    @Test public void combine2Indels() { combine2("CEU.dindel.vcf4.trio.2010_06.indel.genotypes.vcf", "CEU.dindel.vcf4.low_coverage.2010_06.indel.genotypes.vcf", "", "f1c8720fde62687c2e861217670d8b3c"); }
 
     @Test public void combineSNPsAndIndels() { combine2("CEU.trio.2010_03.genotypes.vcf.gz", "CEU.dindel.vcf4.low_coverage.2010_06.indel.genotypes.vcf", "", "e144b6283765494bfe8189ac59965083"); }
 
     @Test public void uniqueSNPs() { combine2("pilot2.snps.vcf4.genotypes.vcf", "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.vcf", "", "89f55abea8f59e39d1effb908440548c"); }
 
-    @Test public void omniHM3Union() { combineSites(" -filteredRecordsMergeType KEEP_IF_ANY_UNFILTERED", "4836086891f6cbdd40eebef3076d215a"); }
-    @Test public void omniHM3Intersect() { combineSites(" -filteredRecordsMergeType KEEP_IF_ALL_UNFILTERED", "6a34b5d743efda8b2f3b639f3a2f5de8"); }
+    @Test public void omniHM3Union() { combineSites(" -filteredRecordsMergeType KEEP_IF_ANY_UNFILTERED", "c6adeda751cb2a08690dd9202356629f"); }
+    @Test public void omniHM3Intersect() { combineSites(" -filteredRecordsMergeType KEEP_IF_ALL_UNFILTERED", "3a08fd5ee18993dfc8882156ccf5d2e9"); }
 
     @Test public void threeWayWithRefs() {
         WalkerTestSpec spec = new WalkerTestSpec(
@@ -110,7 +110,7 @@ public class CombineVariantsIntegrationTest extends WalkerTest {
                         " -priority NA19240_BGI,NA19240_ILLUMINA,NA19240_WUGSC,denovoInfo" +
                         " -genotypeMergeOptions UNIQUIFY -L 1"),
                 1,
-                Arrays.asList("1de95f91ca15d2a8856de35dee0ce33e"));
+                Arrays.asList("b14f8cbb5d03a2e613b12da4da9efd9a"));
         executeTest("threeWayWithRefs", spec);
     }
 
@@ -131,4 +131,13 @@ public class CombineVariantsIntegrationTest extends WalkerTest {
     @Test public void complexTestMinimal() { combineComplexSites(" -minimalVCF", "df96cb3beb2dbb5e02f80abec7d3571e"); }
     @Test public void complexTestSitesOnly() { combineComplexSites(" -sites_only", "f704caeaaaed6711943014b847fe381a"); }
     @Test public void complexTestSitesOnlyMinimal() { combineComplexSites(" -sites_only -minimalVCF", "f704caeaaaed6711943014b847fe381a"); }
+
+    @Test
+    public void combineDBSNPDuplicateSites() {
+         WalkerTestSpec spec = new WalkerTestSpec(
+                 "-T CombineVariants -NO_HEADER -L 1:902000-903000 -o %s -R " + b37KGReference + " -V:v1 " + b37dbSNP132,
+                 1,
+                 Arrays.asList("5969446769cb8377daa2db29304ae6b5"));
+         executeTest("combineDBSNPDuplicateSites:", spec);
+    }
 }
\ No newline at end of file
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
index 20409d4ca..e4ded491b 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/SelectVariantsIntegrationTest.java
@@ -16,7 +16,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         String samplesFile = validationDataLocation + "SelectVariants.samples.txt";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-            baseTestString(" -sn A -se '[CDH]' -sf " + samplesFile + " -env -ef -select 'DP < 250' --variant:VCF3 " + testfile),
+            baseTestString(" -sn A -se '[CDH]' -sf " + samplesFile + " -env -ef -select 'DP < 250' --variant " + testfile),
             1,
             Arrays.asList("d18516c1963802e92cb9e425c0b75fd6")
         );
@@ -30,7 +30,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         String samplesFile = validationDataLocation + "SelectVariants.samples.txt";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-            "-T SelectVariants -R " + b36KGReference + " -L 1:1-1000000 -o %s -NO_HEADER -xl_sn A -xl_sf " + samplesFile + " --variant:VCF3 " + testfile,
+            "-T SelectVariants -R " + b36KGReference + " -L 1:1-1000000 -o %s -NO_HEADER -xl_sn A -xl_sf " + samplesFile + " --variant " + testfile,
             1,
             Arrays.asList("730f021fd6ecf1d195dabbee2e233bfd")
         );
@@ -43,7 +43,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         String testfile = validationDataLocation + "test.dup.vcf";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-                baseTestString(" -sn A -sn B -sn C --variant:VCF3 " + testfile),
+                baseTestString(" -sn A -sn B -sn C --variant " + testfile),
                 1,
                 Arrays.asList("b74038779fe6485dbb8734ae48178356")
         );
@@ -56,7 +56,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         String testFile = validationDataLocation + "NA12878.hg19.example1.vcf";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-                "-T SelectVariants -R " + hg19Reference + " -sn NA12878 -L 20:1012700-1020000 --variant:VCF " + b37hapmapGenotypes + " -disc:VCF " + testFile + " -o %s -NO_HEADER",
+                "-T SelectVariants -R " + hg19Reference + " -sn NA12878 -L 20:1012700-1020000 --variant " + b37hapmapGenotypes + " -disc " + testFile + " -o %s -NO_HEADER",
                 1,
                 Arrays.asList("78e6842325f1f1bc9ab30d5e7737ee6e")
         );
@@ -69,7 +69,7 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         String testFile = validationDataLocation + "NA12878.hg19.example1.vcf";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-                "-T SelectVariants -R " + hg19Reference + " -sn NA12878 -L 20:1012700-1020000 -conc:VCF " + b37hapmapGenotypes + " --variant " + testFile + " -o %s -NO_HEADER",
+                "-T SelectVariants -R " + hg19Reference + " -sn NA12878 -L 20:1012700-1020000 -conc " + b37hapmapGenotypes + " --variant " + testFile + " -o %s -NO_HEADER",
                 1,
                 Arrays.asList("d2ba3ea30a810f6f0fbfb1b643292b6a")
         );
@@ -90,16 +90,16 @@ public class SelectVariantsIntegrationTest extends WalkerTest {
         executeTest("testVariantTypeSelection--" + testFile, spec);
     }
 
-    @Test(enabled=false)
-    public void testRemovePLs() {
+    @Test
+    public void testUsingDbsnpName() {
         String testFile = validationDataLocation + "combine.3.vcf";
 
         WalkerTestSpec spec = new WalkerTestSpec(
-                "-T SelectVariants -R " + b36KGReference + " -sn NA12892 --variant " + testFile + " -o %s -NO_HEADER",
+                "-T SelectVariants -R " + b36KGReference + " -sn NA12892 --variant:dbsnp " + testFile + " -o %s -NO_HEADER",
                 1,
-                Arrays.asList("")
+                Arrays.asList("167a1265df820978a74c267df44d5c43")
         );
 
-        executeTest("testWithPLs--" + testFile, spec);
+        executeTest("testUsingDbsnpName--" + testFile, spec);
     }
 }
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java
index 3801e132d..00044f859 100644
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java
@@ -98,7 +98,7 @@ public class VCFStreamingIntegrationTest extends WalkerTest {
             " -EV CompOverlap -noEV -noST" +
             " -o %s",
             1,
-            Arrays.asList("ea09bf764adba9765b99921c5ba2c709")
+            Arrays.asList("d46a735ffa898f4aa6b3758c5b03f06d")
         );
         executeTest("testVCFStreamingChain", selectTestSpec);
 
diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
index adf3b21a8..5f71f82fd 100755
--- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/ValidateVariantsIntegrationTest.java
@@ -113,4 +113,16 @@ public class ValidateVariantsIntegrationTest extends WalkerTest {
 
         executeTest("test bad alt allele", spec);
     }
+
+    @Test
+    public void testBadAllele2() {
+        WalkerTestSpec spec = new WalkerTestSpec(
+            baseTestString("validationExampleBad3.vcf", "REF"),
+            0,
+            UserException.MalformedFile.class
+        );
+
+        executeTest("test bad ref allele in deletion", spec);
+    }
+
 }
diff --git a/public/java/test/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionIntegrationTest.java b/public/java/test/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionIntegrationTest.java
index 3dfd0550d..48f4c3777 100644
--- a/public/java/test/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionIntegrationTest.java
+++ b/public/java/test/org/broadinstitute/sting/jna/drmaa/v1_0/JnaSessionIntegrationTest.java
@@ -34,6 +34,7 @@ import java.io.File;
 import java.util.*;
 
 public class JnaSessionIntegrationTest extends BaseTest {
+    private String implementation = null;
     private static final SessionFactory factory = new JnaSessionFactory();
 
     @Test
@@ -44,10 +45,23 @@ public class JnaSessionIntegrationTest extends BaseTest {
         System.out.println(String.format("DRMAA contact(s): %s", session.getContact()));
         System.out.println(String.format("DRM system(s): %s", session.getDrmSystem()));
         System.out.println(String.format("DRMAA implementation(s): %s", session.getDrmaaImplementation()));
+        this.implementation = session.getDrmaaImplementation();
     }
 
-    @Test
+    @Test(dependsOnMethods = { "testDrmaa" })
     public void testSubmitEcho() throws Exception {
+        if (implementation.contains("LSF")) {
+            System.err.println("    ***********************************************************");
+            System.err.println("   *************************************************************");
+            System.err.println("   ****                                                     ****");
+            System.err.println("  ****  Skipping JnaSessionIntegrationTest.testSubmitEcho()  ****");
+            System.err.println("  ****      Are you using the dotkit .combined_LSF_SGE?      ****");
+            System.err.println("   ****                                                     ****");
+            System.err.println("   *************************************************************");
+            System.err.println("    ***********************************************************");
+            return;
+        }
+
         File outFile = createNetworkTempFile("JnaSessionIntegrationTest-", ".out");
         Session session = factory.getSession();
         session.init(null);
diff --git a/public/java/test/org/broadinstitute/sting/utils/ReadUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/ReadUtilsUnitTest.java
index 7cb7fec98..e1fdadadc 100755
--- a/public/java/test/org/broadinstitute/sting/utils/ReadUtilsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/ReadUtilsUnitTest.java
@@ -3,6 +3,7 @@ package org.broadinstitute.sting.utils;
 import net.sf.samtools.SAMFileHeader;
 import net.sf.samtools.SAMRecord;
 import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.utils.pileup.PileupElement;
 import org.broadinstitute.sting.utils.sam.ArtificialSAMUtils;
 import org.broadinstitute.sting.utils.sam.ReadUtils;
 import org.testng.Assert;
@@ -12,9 +13,10 @@ import org.testng.annotations.Test;
 
 
 public class ReadUtilsUnitTest extends BaseTest {
-    SAMRecord read;
+    SAMRecord read, reducedRead;
     final static String BASES = "ACTG";
     final static String QUALS = "!+5?";
+    final private static int REDUCED_READ_QUAL = 20;
 
     @BeforeTest
     public void init() {
@@ -23,6 +25,11 @@ public class ReadUtilsUnitTest extends BaseTest {
         read.setReadUnmappedFlag(true);
         read.setReadBases(new String(BASES).getBytes());
         read.setBaseQualityString(new String(QUALS));
+
+        reducedRead = ArtificialSAMUtils.createArtificialRead(header, "reducedRead", 0, 1, BASES.length());
+        reducedRead.setReadBases(BASES.getBytes());
+        reducedRead.setBaseQualityString(QUALS);
+        reducedRead.setAttribute(ReadUtils.REDUCED_READ_QUALITY_TAG, REDUCED_READ_QUAL);
     }
 
     private void testReadBasesAndQuals(SAMRecord read, int expectedStart, int expectedStop) {
@@ -38,4 +45,40 @@ public class ReadUtilsUnitTest extends BaseTest {
     @Test public void testClip2Front() { testReadBasesAndQuals(read, 2, 4); }
     @Test public void testClip1Back() { testReadBasesAndQuals(read, 0, 3); }
     @Test public void testClip2Back() { testReadBasesAndQuals(read, 0, 2); }
+
+    @Test
+    public void testReducedReads() {
+        Assert.assertFalse(ReadUtils.isReducedRead(read), "isReducedRead is false for normal read");
+        Assert.assertEquals(ReadUtils.getReducedReadQualityTagValue(read), null, "No reduced read tag in normal read");
+
+        Assert.assertTrue(ReadUtils.isReducedRead(reducedRead), "isReducedRead is true for reduced read");
+        Assert.assertEquals((int) ReadUtils.getReducedReadQualityTagValue(reducedRead), REDUCED_READ_QUAL, "Reduced read tag is set to expected value");
+    }
+
+    @Test
+    public void testreducedReadWithReducedQualsWithReducedRead() {
+        SAMRecord replacedRead = ReadUtils.reducedReadWithReducedQuals(reducedRead);
+        Assert.assertEquals(replacedRead.getReadBases(), reducedRead.getReadBases());
+        Assert.assertEquals(replacedRead.getBaseQualities().length, reducedRead.getBaseQualities().length);
+        for ( int i = 0; i < replacedRead.getBaseQualities().length; i++)
+            Assert.assertEquals(replacedRead.getBaseQualities()[i], REDUCED_READ_QUAL);
+    }
+
+    @Test(expectedExceptions = IllegalArgumentException.class)
+    public void testreducedReadWithReducedQualsWithNormalRead() {
+        ReadUtils.reducedReadWithReducedQuals(read);
+    }
+
+    @Test
+    public void testReducedReadPileupElement() {
+        PileupElement readp = new PileupElement(read,0);
+        PileupElement reducedreadp = new PileupElement(reducedRead,0);
+
+        Assert.assertFalse(readp.isReducedRead());
+
+        Assert.assertTrue(reducedreadp.isReducedRead());
+        Assert.assertEquals(reducedreadp.getReducedCount(), 0);
+        Assert.assertEquals(reducedreadp.getReducedQual(), REDUCED_READ_QUAL);
+
+    }
 }
diff --git a/public/java/test/org/broadinstitute/sting/utils/baq/BAQUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/baq/BAQUnitTest.java
index 2e4dac6da..67943ccb4 100644
--- a/public/java/test/org/broadinstitute/sting/utils/baq/BAQUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/baq/BAQUnitTest.java
@@ -172,6 +172,17 @@ public class BAQUnitTest extends BaseTest {
         }
     }
 
+    @Test(enabled = true)
+    public void testBAQQualRange() {
+        BAQ baq = new BAQ(1e-3, 0.1, 7, (byte)4, false);         // matches current samtools parameters
+        final byte ref = (byte)'A';
+        final byte alt = (byte)'A';
+
+        for ( int i = 0; i <= SAMUtils.MAX_PHRED_SCORE; i++ )
+            Assert.assertTrue(baq.calcEpsilon( ref, alt, (byte)i) >= 0.0, "Failed to get baq epsilon range");
+    }
+
+
     public void testBAQ(BAQTest test, boolean lookupWithFasta) {
         BAQ baqHMM = new BAQ(1e-3, 0.1, 7, (byte)4, false);         // matches current samtools parameters
 
diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodecUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodecUnitTest.java
deleted file mode 100644
index 6d492565b..000000000
--- a/public/java/test/org/broadinstitute/sting/utils/codecs/snpEff/SnpEffCodecUnitTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Copyright (c) 2011, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.utils.codecs.snpEff;
-
-import org.apache.commons.io.input.ReaderInputStream;
-import org.broad.tribble.TribbleException;
-import org.broad.tribble.readers.AsciiLineReader;
-import org.broad.tribble.readers.LineReader;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.EffectType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.ChangeType;
-import static org.broadinstitute.sting.utils.codecs.snpEff.SnpEffConstants.Zygosity;
-
-import java.io.StringReader;
-
-public class SnpEffCodecUnitTest {
-
-    @Test
-    public void testParseWellFormedSnpEffHeaderLine() {
-        String wellFormedSnpEffHeaderLine = "# Chromo\tPosition\tReference\tChange\tChange type\t" +
-                  "Homozygous\tQuality\tCoverage\tWarnings\tGene_ID\tGene_name\tBio_type\tTrancript_ID\tExon_ID\t" +
-                  "Exon_Rank\tEffect\told_AA/new_AA\tOld_codon/New_codon\tCodon_Num(CDS)\tCDS_size\tCodons around\t" +
-                  "AAs around\tCustom_interval_ID";
-
-        SnpEffCodec codec = new SnpEffCodec();
-        LineReader reader = new AsciiLineReader(new ReaderInputStream(new StringReader(wellFormedSnpEffHeaderLine)));
-        String headerReturned = (String)codec.readHeader(reader);
-
-        Assert.assertEquals(headerReturned, wellFormedSnpEffHeaderLine);
-    }
-
-    @Test(expectedExceptions = TribbleException.InvalidHeader.class)
-    public void testParseWrongNumberOfFieldsSnpEffHeaderLine() {
-        String wrongNumberOfFieldsSnpEffHeaderLine = "# Chromo\tPosition\tReference\tChange\tChange type\t" +
-                  "Homozygous\tQuality\tCoverage\tWarnings\tGene_ID\tGene_name\tBio_type\tTrancript_ID\tExon_ID\t" +
-                  "Exon_Rank\tEffect\told_AA/new_AA\tOld_codon/New_codon\tCodon_Num(CDS)\tCDS_size\tCodons around\t" +
-                  "AAs around";
-
-        SnpEffCodec codec = new SnpEffCodec();
-        LineReader reader = new AsciiLineReader(new ReaderInputStream(new StringReader(wrongNumberOfFieldsSnpEffHeaderLine)));
-        codec.readHeader(reader);
-    }
-
-    @Test(expectedExceptions = TribbleException.InvalidHeader.class)
-    public void testParseMisnamedColumnSnpEffHeaderLine() {
-        String misnamedColumnSnpEffHeaderLine = "# Chromo\tPosition\tRef\tChange\tChange type\t" +
-                  "Homozygous\tQuality\tCoverage\tWarnings\tGene_ID\tGene_name\tBio_type\tTrancript_ID\tExon_ID\t" +
-                  "Exon_Rank\tEffect\told_AA/new_AA\tOld_codon/New_codon\tCodon_Num(CDS)\tCDS_size\tCodons around\t" +
-                  "AAs around\tCustom_interval_ID";
-
-        SnpEffCodec codec = new SnpEffCodec();
-        LineReader reader = new AsciiLineReader(new ReaderInputStream(new StringReader(misnamedColumnSnpEffHeaderLine)));
-        codec.readHeader(reader);
-    }
-
-    @Test
-    public void testParseSimpleEffectSnpEffLine() {
-        String simpleEffectSnpEffLine = "1\t69428\tT\tG\tSNP\tHom\t6049.69\t61573\t\tENSG00000177693\t" +
-                  "OR4F5\tmRNA\tENST00000326183\texon_1_69055_70108\t1\tNON_SYNONYMOUS_CODING\tF/C\tTTT/TGT\t113\t918\t\t\t";
-
-        SnpEffFeature expectedFeature = new SnpEffFeature("1",
-                                                          69428l,
-                                                          "T",
-                                                          "G",
-                                                          ChangeType.SNP,
-                                                          Zygosity.Hom,
-                                                          6049.69,
-                                                          61573l,
-                                                          null,
-                                                          "ENSG00000177693",
-                                                          "OR4F5",
-                                                          "mRNA",
-                                                          "ENST00000326183",
-                                                          "exon_1_69055_70108",
-                                                          1,
-                                                          false,
-                                                          EffectType.NON_SYNONYMOUS_CODING,
-                                                          null,
-                                                          "F/C",
-                                                          "TTT/TGT",
-                                                          113,
-                                                          918,
-                                                          null,
-                                                          null,
-                                                          null
-                                                         );
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(simpleEffectSnpEffLine);
-
-        Assert.assertEquals(feature, expectedFeature);
-    }
-
-    @Test
-    public void testParseNonCodingRegionSnpEffLine() {
-        String nonCodingRegionSnpEffLine = "1\t1337592\tG\tC\tSNP\tHom\t1935.52\t21885\t\tENSG00000250188\t" +
-                  "RP4-758J18.5\tmRNA\tENST00000514958\texon_1_1337454_1338076\t2\tWITHIN_NON_CODING_GENE, NON_SYNONYMOUS_CODING\t" +
-                  "L/V\tCTA/GTA\t272\t952\t\t\t";
-
-        SnpEffFeature expectedFeature = new SnpEffFeature("1",
-                                                          1337592l,
-                                                          "G",
-                                                          "C",
-                                                          ChangeType.SNP,
-                                                          Zygosity.Hom,
-                                                          1935.52,
-                                                          21885l,
-                                                          null,
-                                                          "ENSG00000250188",
-                                                          "RP4-758J18.5",
-                                                          "mRNA",
-                                                          "ENST00000514958",
-                                                          "exon_1_1337454_1338076",
-                                                          2,
-                                                          true,
-                                                          EffectType.NON_SYNONYMOUS_CODING,
-                                                          null,
-                                                          "L/V",
-                                                          "CTA/GTA",
-                                                          272,
-                                                          952,
-                                                          null,
-                                                          null,
-                                                          null
-                                                         );
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(nonCodingRegionSnpEffLine);
-
-        Assert.assertEquals(feature, expectedFeature);
-    }
-
-    @Test
-    public void testParseExtraEffectInformationSnpEffLine() {
-        String extraEffectInformationSnpEffLine = "1\t879537\tT\tC\tSNP\tHom\t341.58\t13733\t\tENSG00000187634\tSAMD11\t" +
-                  "mRNA\tENST00000341065\t\t\tUTR_3_PRIME: 4 bases from transcript end\t\t\t\t\t\t\t";
-
-        SnpEffFeature expectedFeature = new SnpEffFeature("1",
-                                                          879537l,
-                                                          "T",
-                                                          "C",
-                                                          ChangeType.SNP,
-                                                          Zygosity.Hom,
-                                                          341.58,
-                                                          13733l,
-                                                          null,
-                                                          "ENSG00000187634",
-                                                          "SAMD11",
-                                                          "mRNA",
-                                                          "ENST00000341065",
-                                                          null,
-                                                          null,
-                                                          false,
-                                                          EffectType.UTR_3_PRIME,
-                                                          "4 bases from transcript end",
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null
-                                                         );
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(extraEffectInformationSnpEffLine);
-
-        Assert.assertEquals(feature, expectedFeature);
-    }
-
-    @Test
-    public void testParseMultiEffectSnpEffLine() {
-        String multiEffectSnpEffLine = "1\t901901\tC\tT\tSNP\tHom\t162.91\t4646\t\tENSG00000187583\tPLEKHN1\tmRNA\t" +
-                  "ENST00000379410\texon_1_901877_901994\t1\tSTART_GAINED: ATG, UTR_5_PRIME: 11 bases from TSS\t\t\t\t\t\t\t";
-
-        SnpEffFeature expectedFeature = new SnpEffFeature("1",
-                                                          901901l,
-                                                          "C",
-                                                          "T",
-                                                          ChangeType.SNP,
-                                                          Zygosity.Hom,
-                                                          162.91,
-                                                          4646l,
-                                                          null,
-                                                          "ENSG00000187583",
-                                                          "PLEKHN1",
-                                                          "mRNA",
-                                                          "ENST00000379410",
-                                                          "exon_1_901877_901994",
-                                                          1,
-                                                          false,
-                                                          EffectType.START_GAINED,
-                                                          "ATG, UTR_5_PRIME: 11 bases from TSS",
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null,
-                                                          null
-                                                         );
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(multiEffectSnpEffLine);
-
-        Assert.assertEquals(feature, expectedFeature);
-    }
-
-    @Test(expectedExceptions = TribbleException.InvalidDecodeLine.class)
-    public void testParseWrongNumberOfFieldsSnpEffLine() {
-        String wrongNumberOfFieldsSnpEffLine = "1\t69428\tT\tG\tSNP\tHom\t6049.69\t61573\t\tENSG00000177693\t" +
-                  "OR4F5\tmRNA\tENST00000326183\texon_1_69055_70108\t1\tNON_SYNONYMOUS_CODING\tF/C\tTTT/TGT\t113\t918\t\t";
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(wrongNumberOfFieldsSnpEffLine);
-    }
-
-    @Test(expectedExceptions = TribbleException.InvalidDecodeLine.class)
-    public void testParseBlankEffectFieldSnpEffLine() {
-        String blankEffectFieldSnpEffLine = "1\t69428\tT\tG\tSNP\tHom\t6049.69\t61573\t\tENSG00000177693\t" +
-                  "OR4F5\tmRNA\tENST00000326183\texon_1_69055_70108\t1\t\tF/C\tTTT/TGT\t113\t918\t\t\t";
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(blankEffectFieldSnpEffLine);
-    }
-
-    @Test(expectedExceptions = TribbleException.InvalidDecodeLine.class)
-    public void testParseInvalidNumericFieldSnpEffLine() {
-        String invalidNumericFieldSnpEffLine = "1\t69428\tT\tG\tSNP\tHom\t6049.69\t61573\t\tENSG00000177693\t" +
-                  "OR4F5\tmRNA\tENST00000326183\texon_1_69055_70108\t1\tNON_SYNONYMOUS_CODING\tF/C\tTTT/TGT\t113\tfoo\t\t\t";;
-
-        SnpEffCodec codec = new SnpEffCodec();
-        SnpEffFeature feature = (SnpEffFeature)codec.decode(invalidNumericFieldSnpEffLine);
-    }
-}
diff --git a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java
index d08cda949..55bd4783b 100755
--- a/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/codecs/vcf/IndexFactoryUnitTest.java
@@ -1,53 +1,43 @@
 package org.broadinstitute.sting.utils.codecs.vcf;
 
+import net.sf.samtools.SAMSequenceDictionary;
 import org.broad.tribble.Tribble;
 import org.broad.tribble.index.*;
 import org.broad.tribble.iterators.CloseableTribbleIterator;
 import org.broad.tribble.source.BasicFeatureSource;
+import org.broadinstitute.sting.BaseTest;
 import org.broadinstitute.sting.WalkerTest;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import org.testng.Assert;
+import org.testng.annotations.BeforeTest;
 import org.testng.annotations.Test;
 
 import java.io.File;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.*;
 
 /**
  * tests out the various functions in the index factory class
  */
-public class IndexFactoryUnitTest {
+public class IndexFactoryUnitTest extends BaseTest {
 
     File inputFile = new File("public/testdata/HiSeq.10000.vcf");
     File outputFile = new File("public/testdata/onTheFlyOutputTest.vcf");
     File outputFileIndex = Tribble.indexFile(outputFile);
 
-    /**
-     * test out scoring the indexes
-     */
-    @Test
-    public void testScoreIndexes() {
-        /*// make a list of indexes to score
-        Map creators = new HashMap();
-        // add a linear index with the default bin size
-        LinearIndexCreator linearNormal = new LinearIndexCreator();
-        linearNormal.initialize(inputFile, linearNormal.defaultBinSize());
-        creators.add(LInearIndexlinearNormal);
+    private SAMSequenceDictionary dict;
 
-        // create a tree index with a small index size
-        IntervalIndexCreator treeSmallBin = new IntervalIndexCreator();
-        treeSmallBin.initialize(inputFile, Math.max(200,treeSmallBin.defaultBinSize()/10));
-        creators.add(treeSmallBin);
-
-        List indexes = new ArrayList();
-        for (IndexCreator creator : creators)
-            indexes.add(creator.finalizeIndex(0));
-
-        ArrayList scores = IndexFactory.scoreIndexes(0.5,indexes,100, IndexFactory.IndexBalanceApproach.FOR_SEEK_TIME);
-        System.err.println("scores are : ");
-        for (Double score : scores) {
-            System.err.println(score);
-*/
+    @BeforeTest
+    public void setup() {
+        try {
+            dict = new CachingIndexedFastaSequenceFile(new File(b37KGReference)).getSequenceDictionary();
+        }
+        catch(FileNotFoundException ex) {
+            throw new UserException.CouldNotReadInputFile(b37KGReference,ex);
+        }
     }
 
     //
@@ -65,7 +55,7 @@ public class IndexFactoryUnitTest {
             BasicFeatureSource source = new BasicFeatureSource(inputFile.getAbsolutePath(), indexFromInputFile, new VCFCodec());
 
             int counter = 0;
-            VCFWriter writer = new StandardVCFWriter(outputFile);
+            VCFWriter writer = new StandardVCFWriter(outputFile, dict);
             writer.writeHeader((VCFHeader)source.getHeader());
             CloseableTribbleIterator it = source.iterator();
             while (it.hasNext() && (counter++ < maxRecords || maxRecords == -1) ) {
diff --git a/public/java/test/org/broadinstitute/sting/utils/genotype/vcf/VCFWriterUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/genotype/vcf/VCFWriterUnitTest.java
index e3a926fb9..35c6a4993 100644
--- a/public/java/test/org/broadinstitute/sting/utils/genotype/vcf/VCFWriterUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/genotype/vcf/VCFWriterUnitTest.java
@@ -38,12 +38,13 @@ public class VCFWriterUnitTest extends BaseTest {
     private Set additionalColumns = new HashSet();
     private File fakeVCFFile = new File("FAKEVCFFILEFORTESTING.vcf");
     private GenomeLocParser genomeLocParser;
+    private IndexedFastaSequenceFile seq;
 
     @BeforeClass
     public void beforeTests() {
         File referenceFile = new File(hg18Reference);
         try {
-            IndexedFastaSequenceFile seq = new CachingIndexedFastaSequenceFile(referenceFile);
+            seq = new CachingIndexedFastaSequenceFile(referenceFile);
             genomeLocParser = new GenomeLocParser(seq);
         }
         catch(FileNotFoundException ex) {
@@ -55,7 +56,7 @@ public class VCFWriterUnitTest extends BaseTest {
     @Test
     public void testBasicWriteAndRead() {
         VCFHeader header = createFakeHeader(metaData,additionalColumns);
-        VCFWriter writer = new StandardVCFWriter(fakeVCFFile);
+        VCFWriter writer = new StandardVCFWriter(fakeVCFFile, seq.getSequenceDictionary());
         writer.writeHeader(header);
         writer.add(createVC(header));
         writer.add(createVC(header));
@@ -104,7 +105,6 @@ public class VCFWriterUnitTest extends BaseTest {
     public static VCFHeader createFakeHeader(Set metaData, Set additionalColumns) {
         metaData.add(new VCFHeaderLine(VCFHeaderVersion.VCF4_0.getFormatString(), VCFHeaderVersion.VCF4_0.getVersionString()));
         metaData.add(new VCFHeaderLine("two", "2"));
-        additionalColumns.add("FORMAT");
         additionalColumns.add("extra1");
         additionalColumns.add("extra2");
         return new VCFHeader(metaData, additionalColumns);
@@ -158,6 +158,6 @@ public class VCFWriterUnitTest extends BaseTest {
             Assert.assertTrue(additionalColumns.contains(key));
             index++;
         }
-        Assert.assertEquals(index+1, additionalColumns.size()  /* for the header field we don't see */);
+        Assert.assertEquals(index, additionalColumns.size());
     }
 }
diff --git a/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java
index bb892eec8..98b878d23 100644
--- a/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/interval/IntervalUtilsUnitTest.java
@@ -30,6 +30,20 @@ public class IntervalUtilsUnitTest extends BaseTest {
     private SAMFileHeader hg19Header;
     private GenomeLocParser hg19GenomeLocParser;
     private List hg19ReferenceLocs;
+    private List hg19exomeIntervals;
+
+    private List getLocs(String... intervals) {
+        return getLocs(Arrays.asList(intervals));
+    }
+
+    private List getLocs(List intervals) {
+        if (intervals.size() == 0)
+            return hg18ReferenceLocs;
+        List locs = new ArrayList();
+        for (String interval: intervals)
+            locs.add(hg18GenomeLocParser.parseGenomeLoc(interval));
+        return locs;
+    }
 
     @BeforeClass
     public void init() {
@@ -54,12 +68,69 @@ public class IntervalUtilsUnitTest extends BaseTest {
             ReferenceSequenceFile seq = new CachingIndexedFastaSequenceFile(hg19Ref);
             hg19GenomeLocParser = new GenomeLocParser(seq);
             hg19ReferenceLocs = Collections.unmodifiableList(GenomeLocSortedSet.createSetFromSequenceDictionary(referenceDataSource.getReference().getSequenceDictionary()).toList()) ;
+
+            hg19exomeIntervals = Collections.unmodifiableList(IntervalUtils.parseIntervalArguments(hg19GenomeLocParser, Arrays.asList(hg19Intervals), false));
         }
         catch(FileNotFoundException ex) {
             throw new UserException.CouldNotReadInputFile(hg19Ref,ex);
         }
     }
 
+    // -------------------------------------------------------------------------------------
+    //
+    // tests to ensure the quality of the interval cuts of the interval cutting functions
+    //
+    // -------------------------------------------------------------------------------------
+
+    private class IntervalSlicingTest extends TestDataProvider {
+        public int parts;
+        public double maxAllowableVariance;
+
+        private IntervalSlicingTest(final int parts, final double maxAllowableVariance) {
+            super(IntervalSlicingTest.class);
+            this.parts = parts;
+            this.maxAllowableVariance = maxAllowableVariance;
+        }
+
+        public String toString() {
+            return String.format("IntervalSlicingTest parts=%d maxVar=%.2f", parts, maxAllowableVariance);
+        }
+    }
+
+    @DataProvider(name = "intervalslicingdata")
+    public Object[][] createTrees() {
+        new IntervalSlicingTest(1, 0);
+        new IntervalSlicingTest(2, 1);
+        new IntervalSlicingTest(5, 1);
+        new IntervalSlicingTest(10, 1);
+        new IntervalSlicingTest(67, 1);
+        new IntervalSlicingTest(100, 1);
+        new IntervalSlicingTest(500, 1);
+        new IntervalSlicingTest(1000, 1);
+        return IntervalSlicingTest.getTests(IntervalSlicingTest.class);
+    }
+
+    @Test(enabled = true, dataProvider = "intervalslicingdata")
+    public void testFixedScatterIntervalsAlgorithm(IntervalSlicingTest test) {
+        List> splits = IntervalUtils.splitFixedIntervals(hg19exomeIntervals, test.parts);
+
+        long totalSize = IntervalUtils.intervalSize(hg19exomeIntervals);
+        long idealSplitSize = totalSize / test.parts;
+
+        long sumOfSplitSizes = 0;
+        int counter = 0;
+        for ( final List split : splits ) {
+            long splitSize = IntervalUtils.intervalSize(split);
+            double sigma = (splitSize - idealSplitSize) / (1.0 * idealSplitSize);
+            //logger.warn(String.format("Split %d size %d ideal %d sigma %.2f", counter, splitSize, idealSplitSize, sigma));
+            counter++;
+            sumOfSplitSizes += splitSize;
+            Assert.assertTrue(Math.abs(sigma) <= test.maxAllowableVariance, String.format("Interval %d (size %d ideal %d) has a variance %.2f outside of the tolerated range %.2f", counter, splitSize, idealSplitSize, sigma, test.maxAllowableVariance));
+        }
+
+        Assert.assertEquals(totalSize, sumOfSplitSizes, "Split intervals don't contain the exact number of bases in the origianl intervals");
+    }
+
     @Test(expectedExceptions=UserException.class)
     public void testMergeListsBySetOperatorNoOverlap() {
         // a couple of lists we'll use for the testing
@@ -129,19 +200,6 @@ public class IntervalUtilsUnitTest extends BaseTest {
         Assert.assertEquals((long)lengths.get("chrX"), 154913754);
     }
 
-    private List getLocs(String... intervals) {
-        return getLocs(Arrays.asList(intervals));
-    }
-
-    private List getLocs(List intervals) {
-        if (intervals.size() == 0)
-            return hg18ReferenceLocs;
-        List locs = new ArrayList();
-        for (String interval: intervals)
-            locs.add(hg18GenomeLocParser.parseGenomeLoc(interval));
-        return locs;
-    }
-
     @Test
     public void testParseIntervalArguments() {
         Assert.assertEquals(getLocs().size(), 45);
@@ -174,8 +232,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
         List files = testFiles("basic.", 3, ".intervals");
 
         List locs = getLocs("chr1", "chr2", "chr3");
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         List locs1 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(0).toString()), false);
         List locs2 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(1).toString()), false);
@@ -200,8 +258,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
         List files = testFiles("less.", 3, ".intervals");
 
         List locs = getLocs("chr1", "chr2", "chr3", "chr4");
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         List locs1 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(0).toString()), false);
         List locs2 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(1).toString()), false);
@@ -228,8 +286,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
     public void testScatterFixedIntervalsMoreFiles() {
         List files = testFiles("more.", 3, ".intervals");
         List locs = getLocs("chr1", "chr2");
-        List splits = IntervalUtils.splitFixedIntervals(locs, locs.size()); // locs.size() instead of files.size()
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, locs.size()); // locs.size() instead of files.size()
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
     }
     @Test
     public void testScatterFixedIntervalsStart() {
@@ -242,8 +300,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
         List files = testFiles("split.", 3, ".intervals");
 
         List locs = getLocs(intervals);
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         List locs1 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(0).toString()), false);
         List locs2 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(1).toString()), false);
@@ -270,8 +328,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
         List files = testFiles("split.", 3, ".intervals");
 
         List locs = getLocs(intervals);
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         List locs1 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(0).toString()), false);
         List locs2 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(1).toString()), false);
@@ -298,8 +356,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
         List files = testFiles("split.", 3, ".intervals");
 
         List locs = getLocs(intervals);
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         List locs1 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(0).toString()), false);
         List locs2 = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(files.get(1).toString()), false);
@@ -319,7 +377,7 @@ public class IntervalUtilsUnitTest extends BaseTest {
     public void testScatterFixedIntervalsFile() {
         List files = testFiles("sg.", 20, ".intervals");
         List locs = IntervalUtils.parseIntervalArguments(hg18GenomeLocParser, Arrays.asList(BaseTest.GATKDataLocation + "whole_exome_agilent_designed_120.targets.hg18.chr20.interval_list"), false);
-        List splits = IntervalUtils.splitFixedIntervals(locs, files.size());
+        List> splits = IntervalUtils.splitFixedIntervals(locs, files.size());
 
         int[] counts = {
                 125, 138, 287, 291, 312, 105, 155, 324,
@@ -332,16 +390,13 @@ public class IntervalUtilsUnitTest extends BaseTest {
         };
 
         //String splitCounts = "";
-        for (int lastIndex = 0, i = 0; i < splits.size(); i++) {
-            int splitIndex = splits.get(i);
-            int splitCount = (splitIndex - lastIndex);
-            //splitCounts += ", " + splitCount;
-            lastIndex = splitIndex;
+        for (int i = 0; i < splits.size(); i++) {
+            int splitCount = splits.get(i).size();
             Assert.assertEquals(splitCount, counts[i], "Num intervals in split " + i);
         }
         //System.out.println(splitCounts.substring(2));
 
-        IntervalUtils.scatterFixedIntervals(hg18Header, locs, splits, files);
+        IntervalUtils.scatterFixedIntervals(hg18Header, splits, files);
 
         int locIndex = 0;
         for (int i = 0; i < files.size(); i++) {
@@ -357,8 +412,8 @@ public class IntervalUtilsUnitTest extends BaseTest {
     @Test
     public void testScatterFixedIntervalsMax() {
         List files = testFiles("sg.", 85, ".intervals");
-        List splits = IntervalUtils.splitFixedIntervals(hg19ReferenceLocs, files.size());
-        IntervalUtils.scatterFixedIntervals(hg19Header, hg19ReferenceLocs, splits, files);
+        List> splits = IntervalUtils.splitFixedIntervals(hg19ReferenceLocs, files.size());
+        IntervalUtils.scatterFixedIntervals(hg19Header, splits, files);
 
         for (int i = 0; i < files.size(); i++) {
             String file = files.get(i).toString();
diff --git a/public/java/test/org/broadinstitute/sting/utils/variantcontext/GenotypeUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/variantcontext/GenotypeUnitTest.java
new file mode 100644
index 000000000..c4f1efd04
--- /dev/null
+++ b/public/java/test/org/broadinstitute/sting/utils/variantcontext/GenotypeUnitTest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+// our package
+package org.broadinstitute.sting.utils.variantcontext;
+
+
+// the imports for unit testing.
+
+
+import org.broadinstitute.sting.BaseTest;
+import org.testng.Assert;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+
+public class GenotypeUnitTest extends BaseTest {
+    Allele A, Aref, T;
+
+    @BeforeSuite
+    public void before() {
+        A = Allele.create("A");
+        Aref = Allele.create("A", true);
+        T = Allele.create("T");
+    }
+
+//    public Genotype(String sampleName, List alleles, double negLog10PError, Set filters, Map attributes, boolean isPhased) {
+//    public Genotype(String sampleName, List alleles, double negLog10PError, Set filters, Map attributes, boolean isPhased, double[] log10Likelihoods) {
+//    public Genotype(String sampleName, List alleles, double negLog10PError, double[] log10Likelihoods)
+//    public Genotype(String sampleName, List alleles, double negLog10PError)
+//    public Genotype(String sampleName, List alleles)
+//    public List getAlleles()
+//    public List getAlleles(Allele allele)
+//    public Allele getAllele(int i)
+//    public boolean isPhased()
+//    public int getPloidy()
+//    public Type getType()
+//    public boolean isHom()
+//    public boolean isHomRef()
+//    public boolean isHomVar()
+//    public boolean isHet()
+//    public boolean isNoCall()
+//    public boolean isCalled()
+//    public boolean isAvailable()
+//    public boolean hasLikelihoods()
+//    public GenotypeLikelihoods getLikelihoods()
+//    public boolean sameGenotype(Genotype other)
+//    public boolean sameGenotype(Genotype other, boolean ignorePhase)
+//    public String getSampleName()
+//    public boolean hasNegLog10PError()
+//    public double getNegLog10PError()
+//    public double getPhredScaledQual()
+//    public boolean hasAttribute(String key)
+//    public Object getAttribute(String key)
+//    public Object getAttribute(String key, Object defaultValue)
+//    public String getAttributeAsString(String key, String defaultValue)
+//    public int getAttributeAsInt(String key, int defaultValue)
+//    public double getAttributeAsDouble(String key, double  defaultValue)
+//    public boolean getAttributeAsBoolean(String key, boolean  defaultValue)
+}
diff --git a/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUnitTest.java
index f8e6da20a..663eb9ef6 100755
--- a/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUnitTest.java
@@ -5,6 +5,7 @@ package org.broadinstitute.sting.utils.variantcontext;
 // the imports for unit testing.
 
 
+import org.broadinstitute.sting.BaseTest;
 import org.testng.Assert;
 import org.testng.annotations.BeforeSuite;
 import org.testng.annotations.BeforeTest;
@@ -14,10 +15,7 @@ import java.util.Arrays;
 import java.util.List;
 
 
-/**
- * Basic unit test for RecalData
- */
-public class VariantContextUnitTest {
+public class VariantContextUnitTest extends BaseTest {
     Allele A, Aref, T, Tref;
     Allele del, delRef, ATC, ATCref;
 
diff --git a/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUtilsUnitTest.java
new file mode 100644
index 000000000..08db7bcde
--- /dev/null
+++ b/public/java/test/org/broadinstitute/sting/utils/variantcontext/VariantContextUtilsUnitTest.java
@@ -0,0 +1,562 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+package org.broadinstitute.sting.utils.variantcontext;
+
+import net.sf.picard.reference.IndexedFastaSequenceFile;
+import org.broadinstitute.sting.BaseTest;
+import org.broadinstitute.sting.utils.GenomeLocParser;
+import org.broadinstitute.sting.utils.exceptions.UserException;
+import org.broadinstitute.sting.utils.fasta.CachingIndexedFastaSequenceFile;
+import org.testng.Assert;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.*;
+
+public class VariantContextUtilsUnitTest extends BaseTest {
+    Allele Aref, T, C, delRef, Cref, ATC, ATCATC;
+    private GenomeLocParser genomeLocParser;
+
+    @BeforeSuite
+    public void setup() {
+        final File referenceFile = new File(b37KGReference);
+        try {
+            IndexedFastaSequenceFile seq = new CachingIndexedFastaSequenceFile(referenceFile);
+            genomeLocParser = new GenomeLocParser(seq);
+        }
+        catch(FileNotFoundException ex) {
+            throw new UserException.CouldNotReadInputFile(referenceFile,ex);
+        }
+
+        // alleles
+        Aref = Allele.create("A", true);
+        Cref = Allele.create("C", true);
+        delRef = Allele.create("-", true);
+        T = Allele.create("T");
+        C = Allele.create("C");
+        ATC = Allele.create("ATC");
+        ATCATC = Allele.create("ATCATC");
+    }
+
+    private Genotype makeG(String sample, Allele a1, Allele a2) {
+        return new Genotype(sample, Arrays.asList(a1, a2));
+    }
+
+    private Genotype makeG(String sample, Allele a1, Allele a2, double log10pError, double l1, double l2, double l3) {
+        return new Genotype(sample, Arrays.asList(a1, a2), log10pError, new double[]{l1,l2,l3});
+    }
+
+
+    private Genotype makeG(String sample, Allele a1, Allele a2, double log10pError) {
+        return new Genotype(sample, Arrays.asList(a1, a2), log10pError);
+    }
+
+    private VariantContext makeVC(String source, List alleles) {
+        return makeVC(source, alleles, null, null);
+    }
+
+    private VariantContext makeVC(String source, List alleles, Genotype... g1) {
+        return makeVC(source, alleles, Arrays.asList(g1));
+    }
+
+    private VariantContext makeVC(String source, List alleles, String filter) {
+        return makeVC(source, alleles, filter.equals(".") ? null : new HashSet(Arrays.asList(filter)));
+    }
+
+    private VariantContext makeVC(String source, List alleles, Set filters) {
+        return makeVC(source, alleles, null, filters);
+    }
+
+    private VariantContext makeVC(String source, List alleles, Collection genotypes) {
+        return makeVC(source, alleles, genotypes, null);
+    }
+
+    private VariantContext makeVC(String source, List alleles, Collection genotypes, Set filters) {
+        int start = 10;
+        int stop = start; // alleles.contains(ATC) ? start + 3 : start;
+        return new VariantContext(source, "1", start, stop, alleles,
+                genotypes == null ? null : VariantContext.genotypeCollectionToMap(new TreeMap(), genotypes),
+                1.0, filters, null, Cref.getBases()[0]);
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Test allele merging
+    //
+    // --------------------------------------------------------------------------------
+
+    private class MergeAllelesTest extends TestDataProvider {
+        List> inputs;
+        List expected;
+
+        private MergeAllelesTest(List... arg) {
+            super(MergeAllelesTest.class);
+            LinkedList> all = new LinkedList>(Arrays.asList(arg));
+            expected = all.pollLast();
+            inputs = all;
+        }
+
+        public String toString() {
+            return String.format("MergeAllelesTest input=%s expected=%s", inputs, expected);
+        }
+    }
+    @DataProvider(name = "mergeAlleles")
+    public Object[][] mergeAllelesData() {
+        // first, do no harm
+        new MergeAllelesTest(Arrays.asList(Aref),
+                Arrays.asList(Aref));
+
+        new MergeAllelesTest(Arrays.asList(Aref),
+                Arrays.asList(Aref),
+                Arrays.asList(Aref));
+
+        new MergeAllelesTest(Arrays.asList(Aref),
+                Arrays.asList(Aref, T),
+                Arrays.asList(Aref, T));
+
+        new MergeAllelesTest(Arrays.asList(Aref, C),
+                Arrays.asList(Aref, T),
+                Arrays.asList(Aref, C, T));
+
+        new MergeAllelesTest(Arrays.asList(Aref, T),
+                Arrays.asList(Aref, C),
+                Arrays.asList(Aref, C, T)); // sorted by allele
+
+        new MergeAllelesTest(Arrays.asList(Aref, C, T),
+                Arrays.asList(Aref, C),
+                Arrays.asList(Aref, C, T));
+
+        new MergeAllelesTest(Arrays.asList(Aref, T, C),
+                Arrays.asList(Aref, C),
+                Arrays.asList(Aref, C, T)); // sorted by allele
+
+        // The following is actually a pathological case - there's no way on a vcf to represent a null allele that's non-variant.
+        // The code converts this (correctly) to a single-base non-variant vc with whatever base was there as a reference.
+        new MergeAllelesTest(Arrays.asList(delRef),
+                Arrays.asList(Cref));
+
+        new MergeAllelesTest(Arrays.asList(delRef),
+                Arrays.asList(delRef, ATC),
+                Arrays.asList(delRef, ATC));
+
+        new MergeAllelesTest(Arrays.asList(delRef),
+                Arrays.asList(delRef, ATC, ATCATC),
+                Arrays.asList(delRef, ATC, ATCATC));
+
+        new MergeAllelesTest(Arrays.asList(delRef, ATCATC),
+                Arrays.asList(delRef, ATC, ATCATC),
+                Arrays.asList(delRef, ATC, ATCATC));
+
+        new MergeAllelesTest(Arrays.asList(delRef, ATC),
+                Arrays.asList(delRef, ATCATC),
+                Arrays.asList(delRef, ATC, ATCATC));
+
+        return MergeAllelesTest.getTests(MergeAllelesTest.class);
+    }
+
+    @Test(dataProvider = "mergeAlleles")
+    public void testMergeAlleles(MergeAllelesTest cfg) {
+        final List inputs = new ArrayList();
+
+        int i = 0;
+        for ( final List alleles : cfg.inputs ) {
+            final String name = "vcf" + ++i;
+            inputs.add(makeVC(name, alleles));
+        }
+
+        final List priority = vcs2priority(inputs);
+
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                inputs, priority,
+                VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                VariantContextUtils.GenotypeMergeType.PRIORITIZE, false, false, "set", false, false);
+
+        Assert.assertEquals(merged.getAlleles(), cfg.expected);
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Test rsID merging
+    //
+    // --------------------------------------------------------------------------------
+
+    private class SimpleMergeRSIDTest extends TestDataProvider {
+        List inputs;
+        String expected;
+
+        private SimpleMergeRSIDTest(String... arg) {
+            super(SimpleMergeRSIDTest.class);
+            LinkedList allStrings = new LinkedList(Arrays.asList(arg));
+            expected = allStrings.pollLast();
+            inputs = allStrings;
+        }
+
+        public String toString() {
+            return String.format("SimpleMergeRSIDTest vc=%s expected=%s", inputs, expected);
+        }
+    }
+
+    @DataProvider(name = "simplemergersiddata")
+    public Object[][] createSimpleMergeRSIDData() {
+        new SimpleMergeRSIDTest(".", ".");
+        new SimpleMergeRSIDTest(".", ".", ".");
+        new SimpleMergeRSIDTest("rs1", "rs1");
+        new SimpleMergeRSIDTest("rs1", "rs1", "rs1");
+        new SimpleMergeRSIDTest(".", "rs1", "rs1");
+        new SimpleMergeRSIDTest("rs1", ".", "rs1");
+        new SimpleMergeRSIDTest("rs1", "rs2", "rs1,rs2");
+        new SimpleMergeRSIDTest("rs1", "rs2", "rs1", "rs1,rs2"); // duplicates
+        new SimpleMergeRSIDTest("rs2", "rs1", "rs2,rs1");
+        new SimpleMergeRSIDTest("rs2", "rs1", ".", "rs2,rs1");
+        new SimpleMergeRSIDTest("rs2", ".", "rs1", "rs2,rs1");
+        new SimpleMergeRSIDTest("rs1", ".", ".", "rs1");
+        new SimpleMergeRSIDTest("rs1", "rs2", "rs3", "rs1,rs2,rs3");
+
+        return SimpleMergeRSIDTest.getTests(SimpleMergeRSIDTest.class);
+    }
+
+    @Test(dataProvider = "simplemergersiddata")
+    public void testRSIDMerge(SimpleMergeRSIDTest cfg) {
+        final VariantContext snpVC1 = makeVC("snpvc1", Arrays.asList(Aref, T));
+        final List inputs = new ArrayList();
+
+        for ( final String id : cfg.inputs ) {
+            MutableVariantContext vc = new MutableVariantContext(snpVC1);
+            if ( ! id.equals(".") ) vc.setID(id);
+            inputs.add(vc);
+        }
+
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                inputs, null,
+                VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                VariantContextUtils.GenotypeMergeType.UNSORTED, false, false, "set", false, false);
+        Assert.assertEquals(merged.getID(), cfg.expected.equals(".") ? null : cfg.expected);
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Test filtered merging
+    //
+    // --------------------------------------------------------------------------------
+
+    private class MergeFilteredTest extends TestDataProvider {
+        List inputs;
+        VariantContext expected;
+        String setExpected;
+        VariantContextUtils.FilteredRecordMergeType type;
+
+
+        private MergeFilteredTest(String name, VariantContext input1, VariantContext input2, VariantContext expected, String setExpected) {
+            this(name, input1, input2, expected, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED, setExpected);
+        }
+
+        private MergeFilteredTest(String name, VariantContext input1, VariantContext input2, VariantContext expected, VariantContextUtils.FilteredRecordMergeType type, String setExpected) {
+            super(MergeFilteredTest.class, name);
+            LinkedList all = new LinkedList(Arrays.asList(input1, input2));
+            this.expected = expected;
+            this.type = type;
+            inputs = all;
+            this.setExpected = setExpected;
+        }
+
+        public String toString() {
+            return String.format("%s input=%s expected=%s", super.toString(), inputs, expected);
+        }
+    }
+
+    @DataProvider(name = "mergeFiltered")
+    public Object[][] mergeFilteredData() {
+        new MergeFilteredTest("AllPass",
+                makeVC("1", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                makeVC("2", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                makeVC("3", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                VariantContextUtils.MERGE_INTERSECTION);
+
+        new MergeFilteredTest("noFilters",
+                makeVC("1", Arrays.asList(Aref, T), "."),
+                makeVC("2", Arrays.asList(Aref, T), "."),
+                makeVC("3", Arrays.asList(Aref, T), "."),
+                VariantContextUtils.MERGE_INTERSECTION);
+
+        new MergeFilteredTest("oneFiltered",
+                makeVC("1", Arrays.asList(Aref, T), "."),
+                makeVC("2", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("3", Arrays.asList(Aref, T), "."),
+                String.format("1-%s2", VariantContextUtils.MERGE_FILTER_PREFIX));
+
+        new MergeFilteredTest("onePassOneFail",
+                makeVC("1", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                makeVC("2", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("3", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                String.format("1-%s2", VariantContextUtils.MERGE_FILTER_PREFIX));
+
+        new MergeFilteredTest("AllFiltered",
+                makeVC("1", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("2", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("3", Arrays.asList(Aref, T), "FAIL"),
+                VariantContextUtils.MERGE_FILTER_IN_ALL);
+
+        // test ALL vs. ANY
+        new MergeFilteredTest("FailOneUnfiltered",
+                makeVC("1", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("2", Arrays.asList(Aref, T), "."),
+                makeVC("3", Arrays.asList(Aref, T), "."),
+                VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                String.format("%s1-2", VariantContextUtils.MERGE_FILTER_PREFIX));
+
+        new MergeFilteredTest("OneFailAllUnfilteredArg",
+                makeVC("1", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("2", Arrays.asList(Aref, T), "."),
+                makeVC("3", Arrays.asList(Aref, T), "FAIL"),
+                VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ALL_UNFILTERED,
+                String.format("%s1-2", VariantContextUtils.MERGE_FILTER_PREFIX));
+
+        // test excluding allele in filtered record
+        new MergeFilteredTest("DontIncludeAlleleOfFilteredRecords",
+                makeVC("1", Arrays.asList(Aref, T), "."),
+                makeVC("2", Arrays.asList(Aref, T), "FAIL"),
+                makeVC("3", Arrays.asList(Aref, T), "."),
+                String.format("1-%s2", VariantContextUtils.MERGE_FILTER_PREFIX));
+
+        // promotion of site from unfiltered to PASSES
+        new MergeFilteredTest("UnfilteredPlusPassIsPass",
+                makeVC("1", Arrays.asList(Aref, T), "."),
+                makeVC("2", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                makeVC("3", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                VariantContextUtils.MERGE_INTERSECTION);
+
+        new MergeFilteredTest("RefInAll",
+                makeVC("1", Arrays.asList(Aref), VariantContext.PASSES_FILTERS),
+                makeVC("2", Arrays.asList(Aref), VariantContext.PASSES_FILTERS),
+                makeVC("3", Arrays.asList(Aref), VariantContext.PASSES_FILTERS),
+                VariantContextUtils.MERGE_REF_IN_ALL);
+
+        new MergeFilteredTest("RefInOne",
+                makeVC("1", Arrays.asList(Aref), VariantContext.PASSES_FILTERS),
+                makeVC("2", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                makeVC("3", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS),
+                "2");
+
+        return MergeFilteredTest.getTests(MergeFilteredTest.class);
+    }
+
+    @Test(dataProvider = "mergeFiltered")
+    public void testMergeFiltered(MergeFilteredTest cfg) {
+        final List priority = vcs2priority(cfg.inputs);
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                cfg.inputs, priority, cfg.type, VariantContextUtils.GenotypeMergeType.PRIORITIZE, true, false, "set", false, false);
+
+        // test alleles are equal
+        Assert.assertEquals(merged.getAlleles(), cfg.expected.getAlleles());
+
+        // test set field
+        Assert.assertEquals(merged.getAttribute("set"), cfg.setExpected);
+
+        // test filter field
+        Assert.assertEquals(merged.getFilters(), cfg.expected.getFilters());
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Test genotype merging
+    //
+    // --------------------------------------------------------------------------------
+
+    private class MergeGenotypesTest extends TestDataProvider {
+        List inputs;
+        VariantContext expected;
+        List priority;
+
+        private MergeGenotypesTest(String name, String priority, VariantContext... arg) {
+            super(MergeGenotypesTest.class, name);
+            LinkedList all = new LinkedList(Arrays.asList(arg));
+            this.expected = all.pollLast();
+            inputs = all;
+            this.priority = Arrays.asList(priority.split(","));
+        }
+
+        public String toString() {
+            return String.format("%s input=%s expected=%s", super.toString(), inputs, expected);
+        }
+    }
+
+    @DataProvider(name = "mergeGenotypes")
+    public Object[][] mergeGenotypesData() {
+        new MergeGenotypesTest("TakeGenotypeByPriority-1,2", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)));
+
+        new MergeGenotypesTest("TakeGenotypeByPriority-1,2-nocall", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Allele.NO_CALL, Allele.NO_CALL, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Allele.NO_CALL, Allele.NO_CALL, 1)));
+
+        new MergeGenotypesTest("TakeGenotypeByPriority-2,1", "2,1",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2)));
+
+        new MergeGenotypesTest("NonOverlappingGenotypes", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s2", Aref, T, 2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1), makeG("s2", Aref, T, 2)));
+
+        new MergeGenotypesTest("PreserveNoCall", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Allele.NO_CALL, Allele.NO_CALL, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s2", Aref, T, 2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Allele.NO_CALL, Allele.NO_CALL, 1), makeG("s2", Aref, T, 2)));
+
+        new MergeGenotypesTest("PerserveAlleles", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, C), makeG("s2", Aref, C, 2)),
+                makeVC("3", Arrays.asList(Aref, C, T), makeG("s1", Aref, T, 1), makeG("s2", Aref, C, 2)));
+
+        new MergeGenotypesTest("TakeGenotypePartialOverlap-1,2", "1,2",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2), makeG("s3", Aref, T, 3)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1), makeG("s3", Aref, T, 3)));
+
+        new MergeGenotypesTest("TakeGenotypePartialOverlap-2,1", "2,1",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2), makeG("s3", Aref, T, 3)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2), makeG("s3", Aref, T, 3)));
+
+        // merging genothpes with PLs
+        new MergeGenotypesTest("TakeGenotypePartialOverlapWithPLs-2,1", "2,1",
+                makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1,5,0,3)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2,4,0,2), makeG("s3", Aref, T, 3,3,0,2)),
+                makeVC("3", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2,4,0,2), makeG("s3", Aref, T, 3,3,0,2)));
+
+        new MergeGenotypesTest("TakeGenotypePartialOverlapWithPLs-1,2", "1,2",
+                makeVC("1", Arrays.asList(Aref,ATC), makeG("s1", Aref, ATC, 1,5,0,3)),
+                makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2,4,0,2), makeG("s3", Aref, T, 3,3,0,2)),
+                // no likelihoods on result since type changes to mixed multiallelic
+                makeVC("3", Arrays.asList(Aref, ATC, T), makeG("s1", Aref, ATC, 1), makeG("s3", Aref, T, 3)));
+
+        return MergeGenotypesTest.getTests(MergeGenotypesTest.class);
+    }
+
+    @Test(dataProvider = "mergeGenotypes")
+    public void testMergeGenotypes(MergeGenotypesTest cfg) {
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                cfg.inputs, cfg.priority, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                VariantContextUtils.GenotypeMergeType.PRIORITIZE, true, false, "set", false, false);
+
+        // test alleles are equal
+        Assert.assertEquals(merged.getAlleles(), cfg.expected.getAlleles());
+
+        // test genotypes
+        assertGenotypesAreMostlyEqual(merged.getGenotypes(), cfg.expected.getGenotypes());
+    }
+
+    // necessary to not overload equals for genotypes
+    private void assertGenotypesAreMostlyEqual(Map actual, Map expected) {
+        if (actual == expected) {
+            return;
+        }
+
+        if (actual == null || expected == null) {
+            Assert.fail("Maps not equal: expected: " + expected + " and actual: " + actual);
+        }
+
+        if (actual.size() != expected.size()) {
+            Assert.fail("Maps do not have the same size:" + actual.size() + " != " + expected.size());
+        }
+
+        for (Map.Entry entry : actual.entrySet()) {
+            String key = entry.getKey();
+            Genotype value = entry.getValue();
+            Genotype expectedValue = expected.get(key);
+
+            Assert.assertEquals(value.alleles, expectedValue.alleles);
+            Assert.assertEquals(value.getNegLog10PError(), expectedValue.getNegLog10PError());
+            Assert.assertEquals(value.hasLikelihoods(), expectedValue.hasLikelihoods());
+            if ( value.hasLikelihoods() )
+                Assert.assertEquals(value.getLikelihoods().getAsVector(), expectedValue.getLikelihoods().getAsVector());
+        }
+    }
+
+    @Test
+    public void testMergeGenotypesUniquify() {
+        final VariantContext vc1 = makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1));
+        final VariantContext vc2 = makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2));
+
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                Arrays.asList(vc1, vc2), null, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                VariantContextUtils.GenotypeMergeType.UNIQUIFY, false, false, "set", false, false);
+
+        // test genotypes
+        Assert.assertEquals(merged.getGenotypes().keySet(), new HashSet(Arrays.asList("s1.1", "s1.2")));
+    }
+
+    @Test(expectedExceptions = UserException.class)
+    public void testMergeGenotypesRequireUnique() {
+        final VariantContext vc1 = makeVC("1", Arrays.asList(Aref, T), makeG("s1", Aref, T, 1));
+        final VariantContext vc2 = makeVC("2", Arrays.asList(Aref, T), makeG("s1", Aref, T, 2));
+
+        final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                Arrays.asList(vc1, vc2), null, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                VariantContextUtils.GenotypeMergeType.REQUIRE_UNIQUE, false, false, "set", false, false);
+    }
+
+    // --------------------------------------------------------------------------------
+    //
+    // Misc. tests
+    //
+    // --------------------------------------------------------------------------------
+
+    @Test
+    public void testAnnotationSet() {
+        for ( final boolean annotate : Arrays.asList(true, false)) {
+            for ( final String set : Arrays.asList("set", "combine", "x")) {
+                final List priority = Arrays.asList("1", "2");
+                VariantContext vc1 = makeVC("1", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS);
+                VariantContext vc2 = makeVC("2", Arrays.asList(Aref, T), VariantContext.PASSES_FILTERS);
+
+                final VariantContext merged = VariantContextUtils.simpleMerge(genomeLocParser,
+                        Arrays.asList(vc1, vc2), priority, VariantContextUtils.FilteredRecordMergeType.KEEP_IF_ANY_UNFILTERED,
+                        VariantContextUtils.GenotypeMergeType.PRIORITIZE, annotate, false, set, false, false);
+
+                if ( annotate )
+                    Assert.assertEquals(merged.getAttribute(set), VariantContextUtils.MERGE_INTERSECTION);
+                else
+                    Assert.assertFalse(merged.hasAttribute(set));
+            }
+        }
+    }
+
+    private static final List vcs2priority(final Collection vcs) {
+        final List priority = new ArrayList();
+
+        for ( final VariantContext vc : vcs ) {
+            priority.add(vc.getSource());
+        }
+
+        return priority;
+    }
+}
diff --git a/public/packages/AnalyzeCovariates.xml b/public/packages/AnalyzeCovariates.xml
index 7e31934df..e8d58862a 100644
--- a/public/packages/AnalyzeCovariates.xml
+++ b/public/packages/AnalyzeCovariates.xml
@@ -6,10 +6,7 @@
     
       
       
-      
-      
-      
-      
+      
     
   
   
diff --git a/public/packages/FindContaminatingReadGroups.xml b/public/packages/FindContaminatingReadGroups.xml
deleted file mode 100644
index 880f64a81..000000000
--- a/public/packages/FindContaminatingReadGroups.xml
+++ /dev/null
@@ -1,10 +0,0 @@
-
-
-  
-    
-    
-    
-      
-    
-  
-
diff --git a/public/packages/GATKEngine.xml b/public/packages/GATKEngine.xml
index 4f635f7fb..4364988e7 100644
--- a/public/packages/GATKEngine.xml
+++ b/public/packages/GATKEngine.xml
@@ -29,7 +29,7 @@
       
       
       
-      
+      
       
       
       
diff --git a/public/packages/GATKResources.xml b/public/packages/GATKResources.xml
deleted file mode 100755
index 87e6e0e50..000000000
--- a/public/packages/GATKResources.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-  
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-    
-   
-
diff --git a/public/packages/IndelGenotyper.xml b/public/packages/IndelGenotyper.xml
deleted file mode 100644
index c9e3ae0f6..000000000
--- a/public/packages/IndelGenotyper.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-    
-  
-
diff --git a/public/packages/LocalRealignmentAroundIndels.xml b/public/packages/LocalRealignmentAroundIndels.xml
deleted file mode 100644
index 46960e69f..000000000
--- a/public/packages/LocalRealignmentAroundIndels.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-    
-  
-
diff --git a/public/packages/QualityScoresRecalibration.xml b/public/packages/QualityScoresRecalibration.xml
deleted file mode 100644
index 95e8b7c63..000000000
--- a/public/packages/QualityScoresRecalibration.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
-  
-
diff --git a/public/packages/RMDIndexer.xml b/public/packages/RMDIndexer.xml
deleted file mode 100644
index 5d40876de..000000000
--- a/public/packages/RMDIndexer.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-    
-  
-
diff --git a/public/packages/UnifiedGenotyper.xml b/public/packages/UnifiedGenotyper.xml
deleted file mode 100644
index 67a17640c..000000000
--- a/public/packages/UnifiedGenotyper.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-    
-  
-
diff --git a/public/packages/VariantAnnotator.xml b/public/packages/VariantAnnotator.xml
deleted file mode 100644
index 88c0701f0..000000000
--- a/public/packages/VariantAnnotator.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
-  
-
diff --git a/public/packages/VariantEval.xml b/public/packages/VariantEval.xml
deleted file mode 100644
index 791066fb7..000000000
--- a/public/packages/VariantEval.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-      
-      
-      
-      
-      
-    
-  
-
diff --git a/public/packages/VariantFiltration.xml b/public/packages/VariantFiltration.xml
deleted file mode 100644
index 48fa0ff37..000000000
--- a/public/packages/VariantFiltration.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-      
-    
-  
-
diff --git a/public/packages/VariantRecalibration.xml b/public/packages/VariantRecalibration.xml
deleted file mode 100644
index 6fe6b1eff..000000000
--- a/public/packages/VariantRecalibration.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
-  
-    
-    
-    
-      
-      
-      
-    
-  
-
diff --git a/public/perl/liftOverVCF.pl b/public/perl/liftOverVCF.pl
index 21cb8bb6b..ba4198292 100755
--- a/public/perl/liftOverVCF.pl
+++ b/public/perl/liftOverVCF.pl
@@ -36,7 +36,7 @@ my $unsorted_vcf = "$tmp_prefix.unsorted.vcf";
 
 # lift over the file
 print "Lifting over the vcf...";
-my $cmd = "java -jar $gatk/dist/GenomeAnalysisTK.jar -T LiftoverVariants -R $oldRef.fasta -B:variant,vcf $in -o $unsorted_vcf -chain $chain -dict $newRef.dict";
+my $cmd = "java -jar $gatk/dist/GenomeAnalysisTK.jar -T LiftoverVariants -R $oldRef.fasta -V:variant $in -o $unsorted_vcf -chain $chain -dict $newRef.dict";
 if ($recordOriginalLocation) {
   $cmd .= " -recordOriginalLocation";
 }
@@ -66,7 +66,7 @@ system($cmd) == 0 or quit("The sorting step failed.  Please correct the necessar
 
 # Filter the VCF for bad records
 print "\nFixing/removing bad records...\n";
-$cmd = "java -jar $gatk/dist/GenomeAnalysisTK.jar -T FilterLiftedVariants -R $newRef.fasta -B:variant,vcf $sorted_vcf -o $out";
+$cmd = "java -jar $gatk/dist/GenomeAnalysisTK.jar -T FilterLiftedVariants -R $newRef.fasta -V:variant $sorted_vcf -o $out";
 system($cmd) == 0 or quit("The filtering step failed.  Please correct the necessary errors before retrying.");
 
 # clean up
diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
index 59c00b8cd..036a77b58 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala
@@ -131,11 +131,11 @@ class GATKResourcesBundle extends QScript {
     addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/HapMap/3.3/genotypes_r27_nr.b37_fwd.vcf",
       "hapmap_3.3", b37, true, true))
 
-    addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/AFR+EUR+ASN+1KG.dindel_august_release_merged_pilot1.20110126.sites.vcf",
-      "1000G_indels_for_realignment", b37, true, false))
+    addResource(new Resource("/humgen/1kg/processing/official_release/phase1/ALL.wgs.VQSR_consensus_biallelic.20101123.indels.sites.vcf",
+      "1000G_biallelic.indels", b37, true, false))
 
     addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Mills_Devine_Indels_2011/ALL.wgs.indels_mills_devine_hg19_leftAligned_collapsed_double_hit.sites.vcf",
-      "indels_mills_devine", b37, true, true))
+      "Mills_Devine_2hit.indels", b37, true, true))
     
     //
     // example call set for wiki tutorial
@@ -300,9 +300,9 @@ class GATKResourcesBundle extends QScript {
     bamFile = bamIn
   }
 
-  class IndexVCF(@Input vcf: File, @Input ref: File) extends CountRod with UNIVERSAL_GATK_ARGS {
+  class IndexVCF(@Input vcf: File, @Input ref: File) extends CountRODs with UNIVERSAL_GATK_ARGS {
     //@Output val vcfIndex: File = swapExt(vcf.getParent, vcf, ".vcf", ".vcf.idx")
-    this.rodBind :+= RodBind(vcf.getName, "VCF", vcf)
+    this.rod :+= vcf
     this.reference_sequence = ref
   }
 
@@ -313,7 +313,7 @@ class GATKResourcesBundle extends QScript {
   }
 
   class MakeDBSNP129(@Input dbsnp: File, @Input ref: File, @Output dbsnp129: File) extends SelectVariants with UNIVERSAL_GATK_ARGS {
-    this.rodBind :+= RodBind("variant", "VCF", dbsnp)
+    this.variant = dbsnp
     this.select ++= List("\"dbSNPBuildID <= 129\"")
     this.reference_sequence = ref
     this.out = dbsnp129
diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala
index 80bfe03d1..cc4790f05 100755
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/MethodsDevelopmentCallingPipeline.scala
@@ -22,8 +22,8 @@ class MethodsDevelopmentCallingPipeline extends QScript {
   @Argument(shortName="noBAQ", doc="turns off BAQ calculation", required=false)
   var noBAQ: Boolean = false
 
-  @Argument(shortName="indels", doc="calls indels with the Unified Genotyper", required=false)
-  var callIndels: Boolean = false
+  @Argument(shortName="noIndels", doc="do not call indels with the Unified Genotyper", required=false)
+  var noIndels: Boolean = false
 
   @Argument(shortName="LOCAL_ET", doc="Doesn't use the AWS S3 storage for ET option", required=false)
   var LOCAL_ET: Boolean = false
@@ -98,40 +98,52 @@ class MethodsDevelopmentCallingPipeline extends QScript {
   // BUGBUG: We no longer support b36/hg18 because several of the necessary files aren't available aligned to those references
 
   val targetDataSets: Map[String, Target] = Map(
-    "HiSeq" -> new Target("NA12878.HiSeq", hg18, dbSNP_hg18_129, hapmap_hg18,
+    "NA12878_wgs_b37" -> new Target("NA12878.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
+              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.bam"),
+              new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.HiSeq19.filtered.vcf"),
+              "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1),
+    "NA12878_wgs_decoy" -> new Target("NA12878.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
+              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.NA12878.clean.dedup.recal.bam"),
+              new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.HiSeq19.filtered.vcf"),          // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
+              "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1),
+    "NA12878_wgs_hg18" -> new Target("NA12878.HiSeq.WGS.hg18", hg18, dbSNP_hg18_129, hapmap_hg18,
               "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.indels.10.mask",
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.bam"),
               new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/HiSeq.WGS.cleaned.ug.snpfiltered.indelfiltered.vcf"),
               "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg18.intervals", 2.14, 99.0, !lowPass, !exome, 1),
-    "HiSeq19" -> new Target("NA12878.HiSeq19", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
-              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.HiSeq.WGS.bwa.cleaned.recal.hg19.bam"),
-              new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.HiSeq19.filtered.vcf"),
-              "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.noChrY.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1),
-    "GA2hg19" -> new Target("NA12878.GA2.hg19", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
-              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.GA2.WGS.bwa.cleaned.hg19.bam"),
-              new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.GA2.hg19.filtered.vcf"),
-              "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1),
-    "WEx" -> new Target("NA12878.WEx", hg18, dbSNP_hg18_129, hapmap_hg18,
+    "NA12878_wex_b37" -> new Target("NA12878.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
+              new File("/seq/picard_aggregation/C339/NA12878/v3/NA12878.bam"),
+              new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),                  // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
+              "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1),
+    "NA12878_wex_hg18" -> new Target("NA12878.HiSeq.WEx.hg18", hg18, dbSNP_hg18_129, hapmap_hg18,
               "/humgen/gsa-hpprojects/dev/depristo/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.indels.10.mask",
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.WEx.cleaned.recal.bam"),
               new File("/home/radon01/depristo/work/oneOffProjects/1000GenomesProcessingPaper/wgs.v13/GA2.WEx.cleaned.ug.snpfiltered.indelfiltered.vcf"),
               "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1),
-    "WExTrio" -> new Target("CEUTrio.WEx", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
+    "NA12878_wex_decoy" -> new Target("NA12878.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
+              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.NA12878.clean.dedup.recal.bam"),
+              new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),                  // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
+              "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 1),
+    "CEUTrio_wex_b37" -> new Target("CEUTrio.HiSeq.WEx.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.bwa.cleaned.recal.bam"),
               new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),
               "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3),
-    "WGSTrio" -> new Target("CEUTrio.WGS", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
+    "CEUTrio_wgs_b37" -> new Target("CEUTrio.HiSeq.WGS.b37", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.bwa.cleaned.recal.bam"),
               new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),                  // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
               "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3),
-    "WExTrioDecoy" -> new Target("CEUTrio.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
+    "CEUTrio_wex_decoy" -> new Target("CEUTrio.HiSeq.WEx.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WEx.b37_decoy.list"),
               new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),                  // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
               "/seq/references/HybSelOligos/whole_exome_agilent_1.1_refseq_plus_3_boosters/whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list", 3.3, 98.0, !lowPass, exome, 3),
-    "WGSTrioDecoy" -> new Target("CEUTrio.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
+    "CEUTrio_wgs_decoy" -> new Target("CEUTrio.HiSeq.WGS.b37_decoy", b37_decoy, dbSNP_b37, hapmap_b37, indelMask_b37,
               new File("/humgen/gsa-hpprojects/NA12878Collection/bams/CEUTrio.HiSeq.WGS.b37_decoy.list"),
               new File("/humgen/gsa-hpprojects/dev/carneiro/trio/analysis/snps/CEUTrio.WEx.filtered.vcf"),                  // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
               "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.3, 99.0, !lowPass, !exome, 3),
+    "GA2hg19" -> new Target("NA12878.GA2.hg19", hg19, dbSNP_b37, hapmap_b37, indelMask_b37,
+              new File("/humgen/gsa-hpprojects/NA12878Collection/bams/NA12878.GA2.WGS.bwa.cleaned.hg19.bam"),
+              new File("/humgen/gsa-hpprojects/dev/carneiro/hiseq19/analysis/snps/NA12878.GA2.hg19.filtered.vcf"),
+              "/humgen/1kg/processing/pipeline_test_bams/whole_genome_chunked.hg19.intervals", 2.14, 99.0, !lowPass, !exome, 1),
     "FIN" -> new Target("FIN", b37, dbSNP_b37, hapmap_b37, indelMask_b37,
               new File("/humgen/1kg/processing/pipeline_test_bams/FIN.79sample.Nov2010.chr20.bam"),
               new File("/humgen/gsa-hpprojects/dev/data/AugChr20Calls_v4_3state/ALL.august.v4.chr20.filtered.vcf"),         // ** THIS GOLD STANDARD NEEDS TO BE CORRECTED **
@@ -165,7 +177,7 @@ class MethodsDevelopmentCallingPipeline extends QScript {
     val goldStandard = true
     for (target <- targets) {
       if( !skipCalling ) {
-        if (callIndels) add(new indelCall(target), new indelFilter(target), new indelEvaluation(target))
+        if (!noIndels) add(new indelCall(target), new indelFilter(target), new indelEvaluation(target))
         add(new snpCall(target))
         add(new VQSR(target, !goldStandard))
         add(new applyVQSR(target, !goldStandard))
@@ -209,7 +221,7 @@ class MethodsDevelopmentCallingPipeline extends QScript {
       this.max_deletion_fraction = qscript.deletions
     this.out = t.rawVCF
     this.glm = org.broadinstitute.sting.gatk.walkers.genotyper.GenotypeLikelihoodsCalculationModel.Model.SNP
-    this.baq = if (noBAQ) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY}
+    this.baq = if (noBAQ ||  t.isExome) {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.OFF} else {org.broadinstitute.sting.utils.baq.BAQ.CalculationMode.CALCULATE_AS_NECESSARY}
     this.analysisName = t.name + "_UGs"
     this.jobName =  queueLogDir + t.name + ".snpcall"
   }
@@ -248,21 +260,24 @@ class MethodsDevelopmentCallingPipeline extends QScript {
     this.reference_sequence = t.reference
     this.intervalsString ++= List(t.intervals)
     this.input :+= ( if ( goldStandard ) { t.goldStandard_VCF } else { t.rawVCF } )
-    this.training :+= new TaggedFile( t.hapmapFile, "prior=15.0")
-    this.truth :+= new TaggedFile( t.hapmapFile, "prior=15.0")
-    this.training :+= new TaggedFile( omni_b37, "prior=12.0")
-    this.truth :+= new TaggedFile( omni_b37, "prior=12.0")
-    this.training :+= new TaggedFile( training_1000G, "prior=10.0" )
-    this.known :+= new TaggedFile( t.dbsnpFile, "prior=2.0" )
+    this.resource :+= new TaggedFile( t.hapmapFile, "training=true,truth=true,prior=15.0" )
+    this.resource :+= new TaggedFile( omni_b37, "training=true,truth=true,prior=12.0" )
+    this.resource :+= new TaggedFile( training_1000G, "training=true,prior=10.0" )
+    this.resource :+= new TaggedFile( t.dbsnpFile, "known=true,prior=2.0" )
     this.resource :+= new TaggedFile( projectConsensus_1000G, "prior=8.0" )
     this.use_annotation ++= List("QD", "HaplotypeScore", "MQRankSum", "ReadPosRankSum", "MQ", "FS")
-    if(t.nSamples >= 10) {
+    if(t.nSamples >= 10) { // InbreedingCoeff is a population-wide statistic that requires at least 10 samples to calculate
         this.use_annotation ++= List("InbreedingCoeff")
     }
     if(!t.isExome) {
         this.use_annotation ++= List("DP")
-    } else {
+    } else { // exome specific parameters
+        this.resource :+= new TaggedFile( badSites_1000G, "bad=true,prior=2.0" )
         this.mG = 6
+        if(t.nSamples <= 3) { // very few exome samples means very few variants
+            this.mG = 4
+            this.percentBad = 0.04
+        }
     }
     this.tranches_file = if ( goldStandard ) { t.goldStandardTranchesFile } else { t.tranchesFile }
     this.recal_file = if ( goldStandard ) { t.goldStandardRecalFile } else { t.recalFile }
diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala
index 1d473b210..9bddfd97c 100644
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/ExampleUnifiedGenotyper.scala
@@ -56,15 +56,15 @@ class ExampleUnifiedGenotyper extends QScript {
     genotyper.input_file :+= qscript.bamFile
     genotyper.out = swapExt(qscript.bamFile, "bam", "unfiltered.vcf")
 
-    evalUnfiltered.rodBind :+= RodBind("eval", "VCF", genotyper.out)
+    evalUnfiltered.eval :+= genotyper.out
     evalUnfiltered.out = swapExt(genotyper.out, "vcf", "eval")
 
-    variantFilter.rodBind :+= RodBind("variant", "VCF", genotyper.out)
+    variantFilter.variant = genotyper.out
     variantFilter.out = swapExt(qscript.bamFile, "bam", "filtered.vcf")
     variantFilter.filterName = filterNames
     variantFilter.filterExpression = filterExpressions.map("\"" + _ + "\"")
 
-    evalFiltered.rodBind :+= RodBind("eval", "VCF", variantFilter.out)
+    evalFiltered.eval :+= variantFilter.out
     evalFiltered.out = swapExt(variantFilter.out, "vcf", "eval")
 
     add(genotyper, evalUnfiltered)
diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
old mode 100755
new mode 100644
index 297da8cc9..a3e83871e
--- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala
@@ -37,7 +37,7 @@ import org.broadinstitute.sting.utils.exceptions.UserException
 /**
  * Entry point of Queue.  Compiles and runs QScripts passed in to the command line.
  */
-object QCommandLine {
+object QCommandLine extends Logging {
   /**
    * Main.
    * @param argv Arguments.
@@ -45,22 +45,23 @@ object QCommandLine {
   def main(argv: Array[String]) {
     val qCommandLine = new QCommandLine
 
-    Runtime.getRuntime.addShutdownHook(new Thread {
-      /** Cleanup as the JVM shuts down. */
+    val shutdownHook = new Thread {
       override def run() {
+        logger.info("Shutting down jobs. Please wait...")
         ProcessController.shutdown()
         qCommandLine.shutdown()
       }
-    })
+    }
+
+    Runtime.getRuntime.addShutdownHook(shutdownHook)
 
     try {
       CommandLineProgram.start(qCommandLine, argv);
+      Runtime.getRuntime.removeShutdownHook(shutdownHook)
       if (CommandLineProgram.result != 0)
         System.exit(CommandLineProgram.result);
     } catch {
       case e: Exception => CommandLineProgram.exitSystemWithError(e)
-    } finally {
-
     }
   }
 }
diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala
index ef7f2afb0..4cb925d9f 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/engine/FunctionEdge.scala
@@ -156,7 +156,7 @@ class FunctionEdge(val function: QFunction, val inputs: QNode, val outputs: QNod
       val maxLines = 100
       val tailLines = IOUtils.tail(errorFile, maxLines)
       val nl = "%n".format()
-      val summary = if (tailLines.size <= maxLines) "Last %d lines".format(maxLines) else "Contents"
+      val summary = if (tailLines.size > maxLines) "Last %d lines".format(maxLines) else "Contents"
       logger.error("%s of %s:%n%s".format(summary, errorFile, tailLines.mkString(nl)))
     } else {
       logger.error("Unable to access log file: %s".format(errorFile))
diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala
index aae5e438c..0fb997f43 100755
--- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervals.scala
@@ -1,65 +1,65 @@
-/*
- * Copyright (c) 2011, The Broad Institute
- *
- * Permission is hereby granted, free of charge, to any person
- * obtaining a copy of this software and associated documentation
- * files (the "Software"), to deal in the Software without
- * restriction, including without limitation the rights to use,
- * copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following
- * conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
- * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
- * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
- * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- */
-
-package org.broadinstitute.sting.queue.extensions.gatk
-
-import java.io.File
-import collection.JavaConversions._
-import org.broadinstitute.sting.utils.interval.IntervalUtils
-import org.broadinstitute.sting.gatk.datasources.reference.ReferenceDataSource
-import net.sf.samtools.SAMFileHeader
-import java.util.Collections
-import org.broadinstitute.sting.utils.{GenomeLoc, GenomeLocSortedSet, GenomeLocParser}
-
-case class GATKIntervals(reference: File, intervals: List[String]) {
-  private lazy val referenceDataSource = new ReferenceDataSource(reference)
-  private var splitsBySize = Map.empty[Int, java.util.List[java.lang.Integer]]
-
-  lazy val samFileHeader = {
-    val header = new SAMFileHeader
-    header.setSequenceDictionary(referenceDataSource.getReference.getSequenceDictionary)
-    header
-  }
-
-  lazy val locs: java.util.List[GenomeLoc] = {
-    val parser = new GenomeLocParser(referenceDataSource.getReference)
-    val parsedLocs =
-      if (intervals.isEmpty)
-        GenomeLocSortedSet.createSetFromSequenceDictionary(samFileHeader.getSequenceDictionary).toList
-      else
-        IntervalUtils.parseIntervalArguments(parser, intervals, false)
-    Collections.sort(parsedLocs)
-    Collections.unmodifiableList(parsedLocs)
-  }
-
-  lazy val contigs = locs.map(_.getContig).distinct.toList
-
-  def getSplits(size: Int) = {
-    splitsBySize.getOrElse(size, {
-      val splits: java.util.List[java.lang.Integer] = IntervalUtils.splitFixedIntervals(locs, size)
-      splitsBySize += size -> splits
-      splits
-    })
-  }
-}
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.queue.extensions.gatk
+
+import java.io.File
+import collection.JavaConversions._
+import org.broadinstitute.sting.utils.interval.IntervalUtils
+import org.broadinstitute.sting.gatk.datasources.reference.ReferenceDataSource
+import net.sf.samtools.SAMFileHeader
+import java.util.Collections
+import org.broadinstitute.sting.utils.{GenomeLoc, GenomeLocSortedSet, GenomeLocParser}
+
+case class GATKIntervals(reference: File, intervals: List[String]) {
+  private lazy val referenceDataSource = new ReferenceDataSource(reference)
+//  private var splitsBySize = Map.empty[Int, java.util.List[java.lang.Integer]]
+
+  lazy val samFileHeader = {
+    val header = new SAMFileHeader
+    header.setSequenceDictionary(referenceDataSource.getReference.getSequenceDictionary)
+    header
+  }
+
+  lazy val locs: java.util.List[GenomeLoc] = {
+    val parser = new GenomeLocParser(referenceDataSource.getReference)
+    val parsedLocs =
+      if (intervals.isEmpty)
+        GenomeLocSortedSet.createSetFromSequenceDictionary(samFileHeader.getSequenceDictionary).toList
+      else
+        IntervalUtils.parseIntervalArguments(parser, intervals, false)
+    Collections.sort(parsedLocs)
+    Collections.unmodifiableList(parsedLocs)
+  }
+
+  lazy val contigs = locs.map(_.getContig).distinct.toList
+
+//  def getSplits(size: Int) = {
+//    splitsBySize.getOrElse(size, {
+//      val splits: java.util.List[java.lang.Integer] = IntervalUtils.splitFixedIntervals(locs, size)
+//      splitsBySize += size -> splits
+//      splits
+//    })
+//  }
+}
diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala
index d88d272b9..f65d5ab29 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/IntervalScatterFunction.scala
@@ -37,7 +37,7 @@ class IntervalScatterFunction extends GATKScatterFunction with InProcessFunction
 
   def run() {
     val gi = GATKScatterFunction.getGATKIntervals(this.referenceSequence, this.intervals)
-    IntervalUtils.scatterFixedIntervals(gi.samFileHeader, gi.locs,
-      gi.getSplits(this.scatterOutputFiles.size), this.scatterOutputFiles)
+    val splits = IntervalUtils.splitFixedIntervals(gi.locs, this.scatterOutputFiles.size)
+    IntervalUtils.scatterFixedIntervals(gi.samFileHeader, splits, this.scatterOutputFiles)
   }
 }
diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala
index 42f63e225..b4c5d91d3 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/RodBind.scala
@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.function.FileExtension
+import org.broadinstitute.sting.queue.util.FileExtension
 import java.lang.String
 
 /**
diff --git a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala
index ed8158b49..b19f9e430 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/extensions/gatk/TaggedFile.scala
@@ -1,7 +1,7 @@
 package org.broadinstitute.sting.queue.extensions.gatk
 
 import java.io.File
-import org.broadinstitute.sting.queue.function.FileExtension
+import org.broadinstitute.sting.queue.util.FileExtension
 
 /**
  * Used to provide tagged -I input_file arguments to the GATK.
diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
index c905581fa..500f7b200 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
@@ -387,25 +387,11 @@ trait QFunction extends Logging with QJobReport {
    */
   protected def canon(value: Any) = {
     value match {
-      case fileExtension: FileExtension =>
-        val newFile = absolute(fileExtension);
-        val newFileExtension = fileExtension.withPath(newFile.getPath)
-        newFileExtension
-      case file: File =>
-        if (file.getClass != classOf[File])
-          throw new QException("Extensions of file must also extend with FileExtension so that the path can be modified.");
-        absolute(file)
+      case file: File => IOUtils.absolute(commandDirectory, file)
       case x => x
     }
   }
 
-  /**
-   * Returns the absolute path to the file relative to the run directory and the job command directory.
-   * @param file File to root relative to the command directory if it is not already absolute.
-   * @return The absolute path to file.
-   */
-  private def absolute(file: File) = IOUtils.absolute(commandDirectory, file)
-
   /**
    * Scala sugar type for checking annotation required and exclusiveOf.
    */
diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/FileExtension.scala b/public/scala/src/org/broadinstitute/sting/queue/util/FileExtension.scala
similarity index 89%
rename from public/scala/src/org/broadinstitute/sting/queue/function/FileExtension.scala
rename to public/scala/src/org/broadinstitute/sting/queue/util/FileExtension.scala
index e2394a5bf..9b6e52c8e 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/function/FileExtension.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/util/FileExtension.scala
@@ -1,4 +1,4 @@
-package org.broadinstitute.sting.queue.function
+package org.broadinstitute.sting.queue.util
 
 import java.io.File
 
diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala
index 79ffa8cb9..b17ccc0d5 100644
--- a/public/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/util/IOUtils.scala
@@ -3,6 +3,7 @@ package org.broadinstitute.sting.queue.util
 import org.apache.commons.io.FileUtils
 import java.io.{FileReader, File}
 import org.broadinstitute.sting.utils.exceptions.UserException
+import org.broadinstitute.sting.queue.QException
 
 /**
  * A collection of utilities for modifying java.io.
@@ -12,7 +13,7 @@ object IOUtils extends Logging {
    * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
    * @param tempDir Temporary directory.
    */
-  def checkTempDir(tempDir: File) = {
+  def checkTempDir(tempDir: File) {
     val tempDirPath = tempDir.getAbsolutePath
     // Keeps the user from leaving the temp directory as the default, and on Macs from having pluses
     // in the path which can cause problems with the Google Reflections library.
@@ -20,7 +21,7 @@ object IOUtils extends Logging {
     if (tempDirPath.startsWith("/var/folders/") || (tempDirPath == "/tmp") || (tempDirPath == "/tmp/"))
       throw new UserException.BadTmpDir("java.io.tmpdir must be explicitly set")
     if (!tempDir.exists && !tempDir.mkdirs)
-      throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath())
+      throw new UserException.BadTmpDir("Could not create directory: " + tempDir.getAbsolutePath)
   }
 
   /**
@@ -35,9 +36,9 @@ object IOUtils extends Logging {
        throw new UserException.BadTmpDir("Could not create temp directory: " + tempDirParent)
     val temp = File.createTempFile(prefix + "-", suffix, tempDirParent)
     if (!temp.delete)
-      throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath())
+      throw new UserException.BadTmpDir("Could not delete sub file: " + temp.getAbsolutePath)
     if (!temp.mkdir)
-      throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath())
+      throw new UserException.BadTmpDir("Could not create sub directory: " + temp.getAbsolutePath)
     absolute(temp)
   }
 
@@ -46,7 +47,7 @@ object IOUtils extends Logging {
    * @param file File to write to.
    * @param content Content to write.
    */
-  def writeContents(file: File, content: String) =  FileUtils.writeStringToFile(file, content)
+  def writeContents(file: File, content: String) { FileUtils.writeStringToFile(file, content) }
 
   /**
    * Reads content of a file into a string.
@@ -146,10 +147,12 @@ object IOUtils extends Logging {
    * @return The absolute path to the file in the parent dir if the path was not absolute, otherwise the original path.
    */
   def absolute(parent: File, file: File): File = {
-    if (file.isAbsolute)
-      absolute(file)
-    else
-      absolute(new File(parent, file.getPath))
+    val newPath =
+      if (file.isAbsolute)
+        absolutePath(file)
+      else
+        absolutePath(new File(parent, file.getPath))
+    replacePath(file, newPath)
   }
 
   /**
@@ -159,12 +162,16 @@ object IOUtils extends Logging {
    * @return the absolute path to the file.
    */
   def absolute(file: File) = {
+    replacePath(file, absolutePath(file))
+  }
+
+  private def absolutePath(file: File) = {
     var fileAbs = file.getAbsoluteFile
     var names = List.empty[String]
     while (fileAbs != null) {
       val name = fileAbs.getName
       fileAbs = fileAbs.getParentFile
-      
+
       if (name == ".") {
         /* skip */
 
@@ -190,7 +197,18 @@ object IOUtils extends Logging {
       }
     }
 
-    new File(names.mkString("/", "/", ""))
+    names.mkString("/", "/", "")
+  }
+
+  private def replacePath(file: File, path: String) = {
+    file match {
+      case fileExtension: FileExtension =>
+        fileExtension.withPath(path)
+      case file: File =>
+        if (file.getClass != classOf[File])
+          throw new QException("Sub classes of java.io.File must also implement FileExtension so that the path can be modified.")
+        new File(path)
+    }
   }
 
   /**
diff --git a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala
index b3a2d23ae..38abe24ef 100644
--- a/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala
+++ b/public/scala/test/org/broadinstitute/sting/queue/extensions/gatk/GATKIntervalsUnitTest.scala
@@ -53,8 +53,8 @@ class GATKIntervalsUnitTest {
     val gi = new GATKIntervals(hg18Reference, List("chr1:1-1", "chr2:2-3", "chr3:3-5"))
     Assert.assertEquals(gi.locs.toList, List(chr1, chr2, chr3))
     Assert.assertEquals(gi.contigs, List("chr1", "chr2", "chr3"))
-    Assert.assertEquals(gi.getSplits(2).toList, List(2, 3))
-    Assert.assertEquals(gi.getSplits(3).toList, List(1, 2, 3))
+//    Assert.assertEquals(gi.getSplits(2).toList, List(2, 3))
+//    Assert.assertEquals(gi.getSplits(3).toList, List(1, 2, 3))
   }
 
   @Test(timeOut = 30000)
@@ -65,7 +65,7 @@ class GATKIntervalsUnitTest {
     //   for(Item item: javaConvertedScalaList)
     // This for loop is actually an O(N^2) operation as the iterator calls the
     // O(N) javaConvertedScalaList.size() for each iteration of the loop.
-    Assert.assertEquals(gi.getSplits(gi.locs.size).size, 189894)
+    //Assert.assertEquals(gi.getSplits(gi.locs.size).size, 189894)
     Assert.assertEquals(gi.contigs.size, 24)
   }
 
@@ -74,8 +74,8 @@ class GATKIntervalsUnitTest {
     val gi = new GATKIntervals(hg18Reference, Nil)
     Assert.assertEquals(gi.locs, hg18ReferenceLocs)
     Assert.assertEquals(gi.contigs.size, hg18ReferenceLocs.size)
-    Assert.assertEquals(gi.getSplits(2).toList, List(10, 45))
-    Assert.assertEquals(gi.getSplits(4).toList, List(5, 10, 16, 45))
+//    Assert.assertEquals(gi.getSplits(2).toList, List(10, 45))
+//    Assert.assertEquals(gi.getSplits(4).toList, List(5, 10, 16, 45))
   }
 
   @Test
diff --git a/public/testdata/HiSeq.10000.bgzip.vcf.gz b/public/testdata/HiSeq.10000.bgzip.vcf.gz
new file mode 100644
index 000000000..3f2b9bf14
Binary files /dev/null and b/public/testdata/HiSeq.10000.bgzip.vcf.gz differ
diff --git a/settings/repository/org.broad/tribble-24.jar b/settings/repository/org.broad/tribble-25.jar
similarity index 91%
rename from settings/repository/org.broad/tribble-24.jar
rename to settings/repository/org.broad/tribble-25.jar
index b1c39e60a..e260764a5 100644
Binary files a/settings/repository/org.broad/tribble-24.jar and b/settings/repository/org.broad/tribble-25.jar differ
diff --git a/settings/repository/org.broad/tribble-24.xml b/settings/repository/org.broad/tribble-25.xml
similarity index 51%
rename from settings/repository/org.broad/tribble-24.xml
rename to settings/repository/org.broad/tribble-25.xml
index 9b2b967f8..ed7a1fd69 100644
--- a/settings/repository/org.broad/tribble-24.xml
+++ b/settings/repository/org.broad/tribble-25.xml
@@ -1,3 +1,3 @@
 
-    
+