Making the package public, so there's no dependances from public -> private
This commit is contained in:
Mark DePristo
parent
5e593793af
commit
893cc2e103
|
|
@ -0,0 +1,122 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import net.sf.samtools.*;
|
||||||
|
import net.sf.samtools.util.BlockCompressedInputStream;
|
||||||
|
import org.broad.tribble.readers.AsciiLineReader;
|
||||||
|
import org.broad.tribble.readers.LineReader;
|
||||||
|
import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
|
||||||
|
import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
|
||||||
|
import org.broadinstitute.sting.utils.variantcontext.Genotype;
|
||||||
|
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
|
||||||
|
|
||||||
|
import java.io.DataInputStream;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.zip.GZIPInputStream;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 1:09 PM
|
||||||
|
*
|
||||||
|
* Class implementing diffnode reader for VCF
|
||||||
|
*/
|
||||||
|
public class BAMDiffableReader implements DiffableReader {
|
||||||
|
private final static int MAX_RECORDS_TO_READ = 1000;
|
||||||
|
@Override
|
||||||
|
public String getName() { return "BAM"; }
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DiffElement readFromFile(File file) {
|
||||||
|
final SAMFileReader reader = new SAMFileReader(file, null); // null because we don't want it to look for the index
|
||||||
|
reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
|
||||||
|
|
||||||
|
DiffNode root = DiffNode.rooted(file.getName());
|
||||||
|
SAMRecordIterator iterator = reader.iterator();
|
||||||
|
|
||||||
|
int count = 0;
|
||||||
|
while ( iterator.hasNext() ) {
|
||||||
|
if ( count++ > MAX_RECORDS_TO_READ )
|
||||||
|
break;
|
||||||
|
final SAMRecord record = iterator.next();
|
||||||
|
|
||||||
|
// name is the read name + first of pair
|
||||||
|
String name = record.getReadName().replace('.', '_');
|
||||||
|
if ( record.getReadPairedFlag() ) {
|
||||||
|
name += record.getFirstOfPairFlag() ? "_1" : "_2";
|
||||||
|
}
|
||||||
|
|
||||||
|
DiffNode readRoot = DiffNode.empty(name, root);
|
||||||
|
|
||||||
|
// add fields
|
||||||
|
readRoot.add("NAME", record.getReadName());
|
||||||
|
readRoot.add("FLAGS", record.getFlags());
|
||||||
|
readRoot.add("RNAME", record.getReferenceName());
|
||||||
|
readRoot.add("POS", record.getAlignmentStart());
|
||||||
|
readRoot.add("MAPQ", record.getMappingQuality());
|
||||||
|
readRoot.add("CIGAR", record.getCigarString());
|
||||||
|
readRoot.add("RNEXT", record.getMateReferenceName());
|
||||||
|
readRoot.add("PNEXT", record.getMateAlignmentStart());
|
||||||
|
readRoot.add("TLEN", record.getInferredInsertSize());
|
||||||
|
readRoot.add("SEQ", record.getReadString());
|
||||||
|
readRoot.add("QUAL", record.getBaseQualityString());
|
||||||
|
|
||||||
|
for ( SAMRecord.SAMTagAndValue xt : record.getAttributes() ) {
|
||||||
|
readRoot.add(xt.tag, xt.value);
|
||||||
|
}
|
||||||
|
|
||||||
|
// add record to root
|
||||||
|
if ( ! root.hasElement(name) )
|
||||||
|
// protect ourselves from malformed files
|
||||||
|
root.add(readRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
reader.close();
|
||||||
|
|
||||||
|
return root.getBinding();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean canRead(File file) {
|
||||||
|
final byte[] BAM_MAGIC = "BAM\1".getBytes();
|
||||||
|
final byte[] buffer = new byte[BAM_MAGIC.length];
|
||||||
|
try {
|
||||||
|
FileInputStream fstream = new FileInputStream(file);
|
||||||
|
new BlockCompressedInputStream(fstream).read(buffer,0,BAM_MAGIC.length);
|
||||||
|
return Arrays.equals(buffer, BAM_MAGIC);
|
||||||
|
} catch ( IOException e ) {
|
||||||
|
return false;
|
||||||
|
} catch ( net.sf.samtools.FileTruncatedException e ) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,118 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import com.google.java.contract.*;
|
||||||
|
import org.broadinstitute.sting.utils.Utils;
|
||||||
|
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 12:55 PM
|
||||||
|
*
|
||||||
|
* An interface that must be implemented to allow us to calculate differences
|
||||||
|
* between structured objects
|
||||||
|
*/
|
||||||
|
@Invariant({
|
||||||
|
"name != null",
|
||||||
|
"value != null",
|
||||||
|
"parent != null || name.equals(\"ROOT\")",
|
||||||
|
"value == null || value.getBinding() == this"})
|
||||||
|
public class DiffElement {
|
||||||
|
public final static DiffElement ROOT = new DiffElement();
|
||||||
|
|
||||||
|
final private String name;
|
||||||
|
final private DiffElement parent;
|
||||||
|
final private DiffValue value;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For ROOT only
|
||||||
|
*/
|
||||||
|
private DiffElement() {
|
||||||
|
this.name = "ROOT";
|
||||||
|
this.parent = null;
|
||||||
|
this.value = new DiffValue(this, "ROOT");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Requires({"name != null", "parent != null", "value != null"})
|
||||||
|
public DiffElement(String name, DiffElement parent, DiffValue value) {
|
||||||
|
if ( name.equals("ROOT") ) throw new IllegalArgumentException("Cannot use reserved name ROOT");
|
||||||
|
this.name = name;
|
||||||
|
this.parent = parent;
|
||||||
|
this.value = value;
|
||||||
|
this.value.setBinding(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffElement getParent() {
|
||||||
|
return parent;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
public DiffValue getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isRoot() { return this == ROOT; }
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return getName() + "=" + getValue().toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString(int offset) {
|
||||||
|
return (offset > 0 ? Utils.dupString(' ', offset) : 0) + getName() + "=" + getValue().toString(offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
public final String fullyQualifiedName() {
|
||||||
|
if ( isRoot() )
|
||||||
|
return "";
|
||||||
|
else if ( parent.isRoot() )
|
||||||
|
return name;
|
||||||
|
else
|
||||||
|
return parent.fullyQualifiedName() + "." + name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
public String toOneLineString() {
|
||||||
|
return getName() + "=" + getValue().toOneLineString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Ensures({"result != null"})
|
||||||
|
public DiffNode getValueAsNode() {
|
||||||
|
if ( getValue().isCompound() )
|
||||||
|
return (DiffNode)getValue();
|
||||||
|
else
|
||||||
|
throw new ReviewedStingException("Illegal request conversion of a DiffValue into a DiffNode: " + this);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,423 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import com.google.java.contract.Requires;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
import org.broadinstitute.sting.gatk.report.GATKReport;
|
||||||
|
import org.broadinstitute.sting.gatk.report.GATKReportTable;
|
||||||
|
import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
|
||||||
|
import org.broadinstitute.sting.utils.Utils;
|
||||||
|
import org.broadinstitute.sting.utils.classloader.PluginManager;
|
||||||
|
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||||
|
import org.broadinstitute.sting.utils.exceptions.UserException;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.PrintStream;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 12:51 PM
|
||||||
|
* A generic engine for comparing tree-structured objects
|
||||||
|
*/
|
||||||
|
public class DiffEngine {
|
||||||
|
final protected static Logger logger = Logger.getLogger(DiffEngine.class);
|
||||||
|
|
||||||
|
private final Map<String, DiffableReader> readers = new HashMap<String, DiffableReader>();
|
||||||
|
|
||||||
|
public DiffEngine() {
|
||||||
|
loadDiffableReaders();
|
||||||
|
}
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// difference calculation
|
||||||
|
//
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public List<Difference> diff(DiffElement master, DiffElement test) {
|
||||||
|
DiffValue masterValue = master.getValue();
|
||||||
|
DiffValue testValue = test.getValue();
|
||||||
|
|
||||||
|
if ( masterValue.isCompound() && masterValue.isCompound() ) {
|
||||||
|
return diff(master.getValueAsNode(), test.getValueAsNode());
|
||||||
|
} else if ( masterValue.isAtomic() && testValue.isAtomic() ) {
|
||||||
|
return diff(masterValue, testValue);
|
||||||
|
} else {
|
||||||
|
// structural difference in types. one is node, other is leaf
|
||||||
|
return Arrays.asList(new Difference(master, test));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Difference> diff(DiffNode master, DiffNode test) {
|
||||||
|
Set<String> allNames = new HashSet<String>(master.getElementNames());
|
||||||
|
allNames.addAll(test.getElementNames());
|
||||||
|
List<Difference> diffs = new ArrayList<Difference>();
|
||||||
|
|
||||||
|
for ( String name : allNames ) {
|
||||||
|
DiffElement masterElt = master.getElement(name);
|
||||||
|
DiffElement testElt = test.getElement(name);
|
||||||
|
if ( masterElt == null && testElt == null ) {
|
||||||
|
throw new ReviewedStingException("BUG: unexceptedly got two null elements for field: " + name);
|
||||||
|
} else if ( masterElt == null || testElt == null ) { // if either is null, we are missing a value
|
||||||
|
// todo -- should one of these be a special MISSING item?
|
||||||
|
diffs.add(new Difference(masterElt, testElt));
|
||||||
|
} else {
|
||||||
|
diffs.addAll(diff(masterElt, testElt));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return diffs;
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<Difference> diff(DiffValue master, DiffValue test) {
|
||||||
|
if ( master.getValue().equals(test.getValue()) ) {
|
||||||
|
return Collections.emptyList();
|
||||||
|
} else {
|
||||||
|
return Arrays.asList(new Difference(master.getBinding(), test.getBinding()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Summarizing differences
|
||||||
|
//
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Emits a summary of the diffs to out. Suppose you have the following three differences:
|
||||||
|
*
|
||||||
|
* A.X.Z:1!=2
|
||||||
|
* A.Y.Z:3!=4
|
||||||
|
* B.X.Z:5!=6
|
||||||
|
*
|
||||||
|
* The above is the itemized list of the differences. The summary looks for common differences
|
||||||
|
* in the name hierarchy, counts those shared elements, and emits the differences that occur
|
||||||
|
* in order of decreasing counts.
|
||||||
|
*
|
||||||
|
* So, in the above example, what are the shared elements?
|
||||||
|
*
|
||||||
|
* A.X.Z and B.X.Z share X.Z, so there's a *.X.Z with count 2
|
||||||
|
* A.X.Z, A.Y.Z, and B.X.Z all share *.*.Z, with count 3
|
||||||
|
* Each of A.X.Z, A.Y.Z, and B.X.Z are individually unique, with count 1
|
||||||
|
*
|
||||||
|
* So we would emit the following summary:
|
||||||
|
*
|
||||||
|
* *.*.Z: 3
|
||||||
|
* *.X.Z: 2
|
||||||
|
* A.X.Z: 1 [specific difference: 1!=2]
|
||||||
|
* A.Y.Z: 1 [specific difference: 3!=4]
|
||||||
|
* B.X.Z: 1 [specific difference: 5!=6]
|
||||||
|
*
|
||||||
|
* The algorithm to accomplish this calculation is relatively simple. Start with all of the
|
||||||
|
* concrete differences. For each pair of differences A1.A2....AN and B1.B2....BN:
|
||||||
|
*
|
||||||
|
* find the longest common subsequence Si.Si+1...SN where Ai = Bi = Si
|
||||||
|
* If i == 0, then there's no shared substructure
|
||||||
|
* If i > 0, then generate the summarized value X = *.*...Si.Si+1...SN
|
||||||
|
* if X is a known summary, increment it's count, otherwise set its count to 1
|
||||||
|
*
|
||||||
|
* Not that only pairs of the same length are considered as potentially equivalent
|
||||||
|
*
|
||||||
|
* @param params determines how we display the items
|
||||||
|
* @param diffs
|
||||||
|
*/
|
||||||
|
public void reportSummarizedDifferences(List<Difference> diffs, SummaryReportParams params ) {
|
||||||
|
printSummaryReport(summarizeDifferences(diffs), params );
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<SummarizedDifference> summarizeDifferences(List<Difference> diffs) {
|
||||||
|
List<String[]> diffPaths = new ArrayList<String[]>(diffs.size());
|
||||||
|
|
||||||
|
for ( Difference diff1 : diffs ) {
|
||||||
|
diffPaths.add(diffNameToPath(diff1.getFullyQualifiedName()));
|
||||||
|
}
|
||||||
|
|
||||||
|
return summarizedDifferencesOfPaths(diffPaths);
|
||||||
|
}
|
||||||
|
|
||||||
|
final protected static String[] diffNameToPath(String diffName) {
|
||||||
|
return diffName.split("\\.");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected List<SummarizedDifference> summarizedDifferencesOfPaths(List<String[]> diffPaths) {
|
||||||
|
Map<String, SummarizedDifference> summaries = new HashMap<String, SummarizedDifference>();
|
||||||
|
|
||||||
|
// create the initial set of differences
|
||||||
|
for ( int i = 0; i < diffPaths.size(); i++ ) {
|
||||||
|
for ( int j = 0; j <= i; j++ ) {
|
||||||
|
String[] diffPath1 = diffPaths.get(i);
|
||||||
|
String[] diffPath2 = diffPaths.get(j);
|
||||||
|
if ( diffPath1.length == diffPath2.length ) {
|
||||||
|
int lcp = longestCommonPostfix(diffPath1, diffPath2);
|
||||||
|
String path = lcp > 0 ? summarizedPath(diffPath2, lcp) : Utils.join(".", diffPath2);
|
||||||
|
addSummary(summaries, path, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// count differences
|
||||||
|
for ( String[] diffPath : diffPaths ) {
|
||||||
|
for ( SummarizedDifference sumDiff : summaries.values() ) {
|
||||||
|
if ( sumDiff.matches(diffPath) )
|
||||||
|
addSummary(summaries, sumDiff.getPath(), false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<SummarizedDifference> sortedSummaries = new ArrayList<SummarizedDifference>(summaries.values());
|
||||||
|
Collections.sort(sortedSummaries);
|
||||||
|
return sortedSummaries;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void addSummary(Map<String, SummarizedDifference> summaries, String path, boolean onlyCatalog) {
|
||||||
|
if ( summaries.containsKey(path) ) {
|
||||||
|
if ( ! onlyCatalog )
|
||||||
|
summaries.get(path).incCount();
|
||||||
|
} else {
|
||||||
|
SummarizedDifference sumDiff = new SummarizedDifference(path);
|
||||||
|
summaries.put(sumDiff.getPath(), sumDiff);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void printSummaryReport(List<SummarizedDifference> sortedSummaries, SummaryReportParams params ) {
|
||||||
|
GATKReport report = new GATKReport();
|
||||||
|
final String tableName = "diffences";
|
||||||
|
report.addTable(tableName, "Summarized differences between the master and test files.\nSee http://www.broadinstitute.org/gsa/wiki/index.php/DiffObjectsWalker_and_SummarizedDifferences for more information");
|
||||||
|
GATKReportTable table = report.getTable(tableName);
|
||||||
|
table.addPrimaryKey("Difference", true);
|
||||||
|
table.addColumn("NumberOfOccurrences", 0);
|
||||||
|
|
||||||
|
int count = 0, count1 = 0;
|
||||||
|
for ( SummarizedDifference diff : sortedSummaries ) {
|
||||||
|
if ( diff.getCount() < params.minSumDiffToShow )
|
||||||
|
// in order, so break as soon as the count is too low
|
||||||
|
break;
|
||||||
|
|
||||||
|
if ( params.maxItemsToDisplay != 0 && count++ > params.maxItemsToDisplay )
|
||||||
|
break;
|
||||||
|
|
||||||
|
if ( diff.getCount() == 1 ) {
|
||||||
|
count1++;
|
||||||
|
if ( params.maxCountOneItems != 0 && count1 > params.maxCountOneItems )
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
table.set(diff.getPath(), "NumberOfOccurrences", diff.getCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
table.write(params.out);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static int longestCommonPostfix(String[] diffPath1, String[] diffPath2) {
|
||||||
|
int i = 0;
|
||||||
|
for ( ; i < diffPath1.length; i++ ) {
|
||||||
|
int j = diffPath1.length - i - 1;
|
||||||
|
if ( ! diffPath1[j].equals(diffPath2[j]) )
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* parts is [A B C D]
|
||||||
|
* commonPostfixLength: how many parts are shared at the end, suppose its 2
|
||||||
|
* We want to create a string *.*.C.D
|
||||||
|
*
|
||||||
|
* @param parts
|
||||||
|
* @param commonPostfixLength
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
protected static String summarizedPath(String[] parts, int commonPostfixLength) {
|
||||||
|
int stop = parts.length - commonPostfixLength;
|
||||||
|
if ( stop > 0 ) parts = parts.clone();
|
||||||
|
for ( int i = 0; i < stop; i++ ) {
|
||||||
|
parts[i] = "*";
|
||||||
|
}
|
||||||
|
return Utils.join(".", parts);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TODO -- all of the algorithms above should use SummarizedDifference instead
|
||||||
|
* TODO -- of some SummarizedDifferences and some low-level String[]
|
||||||
|
*/
|
||||||
|
public static class SummarizedDifference implements Comparable<SummarizedDifference> {
|
||||||
|
final String path; // X.Y.Z
|
||||||
|
final String[] parts;
|
||||||
|
int count = 0;
|
||||||
|
|
||||||
|
public SummarizedDifference(String path) {
|
||||||
|
this.path = path;
|
||||||
|
this.parts = diffNameToPath(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void incCount() { count++; }
|
||||||
|
|
||||||
|
public int getCount() {
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The fully qualified path object A.B.C etc
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public String getPath() {
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @return the length of the parts of this summary
|
||||||
|
*/
|
||||||
|
public int length() {
|
||||||
|
return this.parts.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true if the string parts matches this summary. Matches are
|
||||||
|
* must be equal() everywhere where this summary isn't *.
|
||||||
|
* @param otherParts
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean matches(String[] otherParts) {
|
||||||
|
if ( otherParts.length != length() )
|
||||||
|
return false;
|
||||||
|
|
||||||
|
// TODO optimization: can start at right most non-star element
|
||||||
|
for ( int i = 0; i < length(); i++ ) {
|
||||||
|
String part = parts[i];
|
||||||
|
if ( ! part.equals("*") && ! part.equals(otherParts[i]) )
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return String.format("%s:%d", getPath(), getCount());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int compareTo(SummarizedDifference other) {
|
||||||
|
// sort first highest to lowest count, then by lowest to highest path
|
||||||
|
int countCmp = Integer.valueOf(count).compareTo(other.count);
|
||||||
|
return countCmp != 0 ? -1 * countCmp : path.compareTo(other.path);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// plugin manager
|
||||||
|
//
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public void loadDiffableReaders() {
|
||||||
|
List<Class<? extends DiffableReader>> drClasses = new PluginManager<DiffableReader>( DiffableReader.class ).getPlugins();
|
||||||
|
|
||||||
|
logger.info("Loading diffable modules:");
|
||||||
|
for (Class<? extends DiffableReader> drClass : drClasses ) {
|
||||||
|
logger.info("\t" + drClass.getSimpleName());
|
||||||
|
|
||||||
|
try {
|
||||||
|
DiffableReader dr = drClass.newInstance();
|
||||||
|
readers.put(dr.getName(), dr);
|
||||||
|
} catch (InstantiationException e) {
|
||||||
|
throw new ReviewedStingException("Unable to instantiate module '" + drClass.getSimpleName() + "'");
|
||||||
|
} catch (IllegalAccessException e) {
|
||||||
|
throw new ReviewedStingException("Illegal access error when trying to instantiate '" + drClass.getSimpleName() + "'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, DiffableReader> getReaders() {
|
||||||
|
return readers;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected DiffableReader getReader(String name) {
|
||||||
|
return readers.get(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a reader appropriate for this file, or null if no such reader exists
|
||||||
|
* @param file
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public DiffableReader findReaderForFile(File file) {
|
||||||
|
for ( DiffableReader reader : readers.values() )
|
||||||
|
if (reader.canRead(file) )
|
||||||
|
return reader;
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true if reader appropriate for this file, or false if no such reader exists
|
||||||
|
* @param file
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean canRead(File file) {
|
||||||
|
return findReaderForFile(file) != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffElement createDiffableFromFile(File file) {
|
||||||
|
DiffableReader reader = findReaderForFile(file);
|
||||||
|
if ( reader == null )
|
||||||
|
throw new UserException("Unsupported file type: " + file);
|
||||||
|
else
|
||||||
|
return reader.readFromFile(file);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean simpleDiffFiles(File masterFile, File testFile, DiffEngine.SummaryReportParams params) {
|
||||||
|
DiffEngine diffEngine = new DiffEngine();
|
||||||
|
|
||||||
|
if ( diffEngine.canRead(masterFile) && diffEngine.canRead(testFile) ) {
|
||||||
|
DiffElement master = diffEngine.createDiffableFromFile(masterFile);
|
||||||
|
DiffElement test = diffEngine.createDiffableFromFile(testFile);
|
||||||
|
List<Difference> diffs = diffEngine.diff(master, test);
|
||||||
|
diffEngine.reportSummarizedDifferences(diffs, params);
|
||||||
|
return true;
|
||||||
|
} else {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static class SummaryReportParams {
|
||||||
|
PrintStream out = System.out;
|
||||||
|
int maxItemsToDisplay = 0;
|
||||||
|
int maxCountOneItems = 0;
|
||||||
|
int minSumDiffToShow = 0;
|
||||||
|
|
||||||
|
public SummaryReportParams(PrintStream out, int maxItemsToDisplay, int maxCountOneItems, int minSumDiffToShow) {
|
||||||
|
this.out = out;
|
||||||
|
this.maxItemsToDisplay = maxItemsToDisplay;
|
||||||
|
this.maxCountOneItems = maxCountOneItems;
|
||||||
|
this.minSumDiffToShow = minSumDiffToShow;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,239 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import com.google.java.contract.Requires;
|
||||||
|
import org.broadinstitute.sting.utils.Utils;
|
||||||
|
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 12:55 PM
|
||||||
|
*
|
||||||
|
* An interface that must be implemented to allow us to calculate differences
|
||||||
|
* between structured objects
|
||||||
|
*/
|
||||||
|
public class DiffNode extends DiffValue {
|
||||||
|
private Map<String, DiffElement> getElementMap() {
|
||||||
|
return (Map<String, DiffElement>)super.getValue();
|
||||||
|
}
|
||||||
|
private static Map<String, DiffElement> emptyElements() { return new HashMap<String, DiffElement>(); }
|
||||||
|
|
||||||
|
private DiffNode(Map<String, DiffElement> elements) {
|
||||||
|
super(elements);
|
||||||
|
}
|
||||||
|
|
||||||
|
private DiffNode(DiffElement binding, Map<String, DiffElement> elements) {
|
||||||
|
super(binding, elements);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// constructors
|
||||||
|
//
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public static DiffNode rooted(String name) {
|
||||||
|
return empty(name, DiffElement.ROOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DiffNode empty(String name, DiffElement parent) {
|
||||||
|
DiffNode df = new DiffNode(emptyElements());
|
||||||
|
DiffElement elt = new DiffElement(name, parent, df);
|
||||||
|
df.setBinding(elt);
|
||||||
|
return df;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DiffNode empty(String name, DiffValue parent) {
|
||||||
|
return empty(name, parent.getBinding());
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// accessors
|
||||||
|
//
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAtomic() { return false; }
|
||||||
|
|
||||||
|
public Collection<String> getElementNames() {
|
||||||
|
return getElementMap().keySet();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Collection<DiffElement> getElements() {
|
||||||
|
return getElementMap().values();
|
||||||
|
}
|
||||||
|
|
||||||
|
private Collection<DiffElement> getElements(boolean atomicOnly) {
|
||||||
|
List<DiffElement> elts = new ArrayList<DiffElement>();
|
||||||
|
for ( DiffElement elt : getElements() )
|
||||||
|
if ( (atomicOnly && elt.getValue().isAtomic()) || (! atomicOnly && elt.getValue().isCompound()))
|
||||||
|
elts.add(elt);
|
||||||
|
return elts;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Collection<DiffElement> getAtomicElements() {
|
||||||
|
return getElements(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Collection<DiffElement> getCompoundElements() {
|
||||||
|
return getElements(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffElement getElement(String name) {
|
||||||
|
for ( DiffElement elt : getElements() )
|
||||||
|
if ( elt.getName().equals(name) )
|
||||||
|
return elt;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true if name is bound in this node
|
||||||
|
* @param name
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public boolean hasElement(String name) {
|
||||||
|
return getElement(name) != null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// add
|
||||||
|
//
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@Requires("elt != null")
|
||||||
|
public void add(DiffElement elt) {
|
||||||
|
if ( getElementMap().containsKey(elt.getName()) )
|
||||||
|
throw new IllegalArgumentException("Attempting to rebind already existing binding: " + elt + " node=" + this);
|
||||||
|
getElementMap().put(elt.getName(), elt);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Requires("elt != null")
|
||||||
|
public void add(DiffValue elt) {
|
||||||
|
add(elt.getBinding());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Requires("elts != null")
|
||||||
|
public void add(Collection<DiffElement> elts) {
|
||||||
|
for ( DiffElement e : elts )
|
||||||
|
add(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void add(String name, Object value) {
|
||||||
|
add(new DiffElement(name, this.getBinding(), new DiffValue(value)));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// toString
|
||||||
|
//
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return toString(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString(int offset) {
|
||||||
|
String off = offset > 0 ? Utils.dupString(' ', offset) : "";
|
||||||
|
StringBuilder b = new StringBuilder();
|
||||||
|
|
||||||
|
b.append("(").append("\n");
|
||||||
|
Collection<DiffElement> atomicElts = getAtomicElements();
|
||||||
|
for ( DiffElement elt : atomicElts ) {
|
||||||
|
b.append(elt.toString(offset + 2)).append('\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
for ( DiffElement elt : getCompoundElements() ) {
|
||||||
|
b.append(elt.toString(offset + 4)).append('\n');
|
||||||
|
}
|
||||||
|
b.append(off).append(")").append("\n");
|
||||||
|
|
||||||
|
return b.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toOneLineString() {
|
||||||
|
StringBuilder b = new StringBuilder();
|
||||||
|
|
||||||
|
b.append('(');
|
||||||
|
List<String> parts = new ArrayList<String>();
|
||||||
|
for ( DiffElement elt : getElements() )
|
||||||
|
parts.add(elt.toOneLineString());
|
||||||
|
b.append(Utils.join(" ", parts));
|
||||||
|
b.append(')');
|
||||||
|
|
||||||
|
return b.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// fromString and toOneLineString
|
||||||
|
//
|
||||||
|
// --------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
public static DiffElement fromString(String tree) {
|
||||||
|
return fromString(tree, DiffElement.ROOT);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Doesn't support full tree structure parsing
|
||||||
|
* @param tree
|
||||||
|
* @param parent
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
private static DiffElement fromString(String tree, DiffElement parent) {
|
||||||
|
// X=(A=A B=B C=(D=D))
|
||||||
|
String[] parts = tree.split("=", 2);
|
||||||
|
if ( parts.length != 2 )
|
||||||
|
throw new ReviewedStingException("Unexpected tree structure: " + tree + " parts=" + parts);
|
||||||
|
String name = parts[0];
|
||||||
|
String value = parts[1];
|
||||||
|
|
||||||
|
if ( value.length() == 0 )
|
||||||
|
throw new ReviewedStingException("Illegal tree structure: " + value + " at " + tree);
|
||||||
|
|
||||||
|
if ( value.charAt(0) == '(' ) {
|
||||||
|
if ( ! value.endsWith(")") )
|
||||||
|
throw new ReviewedStingException("Illegal tree structure. Missing ): " + value + " at " + tree);
|
||||||
|
String subtree = value.substring(1, value.length()-1);
|
||||||
|
DiffNode rec = DiffNode.empty(name, parent);
|
||||||
|
String[] subParts = subtree.split(" ");
|
||||||
|
for ( String subPart : subParts ) {
|
||||||
|
rec.add(fromString(subPart, rec.getBinding()));
|
||||||
|
}
|
||||||
|
return rec.getBinding();
|
||||||
|
} else {
|
||||||
|
return new DiffValue(name, parent, value).getBinding();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,113 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import org.apache.xmlbeans.impl.tool.Diff;
|
||||||
|
import org.broadinstitute.sting.commandline.Argument;
|
||||||
|
import org.broadinstitute.sting.commandline.Output;
|
||||||
|
import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
|
||||||
|
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
|
||||||
|
import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
|
||||||
|
import org.broadinstitute.sting.gatk.walkers.Requires;
|
||||||
|
import org.broadinstitute.sting.gatk.walkers.RodWalker;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.PrintStream;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Compares two record-oriented files, itemizing specific difference between equivalent
|
||||||
|
* records in the two files. Reports both itemized and summarized differences.
|
||||||
|
* @author Mark DePristo
|
||||||
|
* @version 0.1
|
||||||
|
*/
|
||||||
|
@Requires(value={})
|
||||||
|
public class DiffObjectsWalker extends RodWalker<Integer, Integer> {
|
||||||
|
@Output(doc="File to which results should be written",required=true)
|
||||||
|
protected PrintStream out;
|
||||||
|
|
||||||
|
@Argument(fullName="maxRecords", shortName="M", doc="Max. number of records to process", required=false)
|
||||||
|
int MAX_RECORDS = 0;
|
||||||
|
|
||||||
|
@Argument(fullName="maxCount1Records", shortName="M1", doc="Max. number of records occuring exactly once in the file to process", required=false)
|
||||||
|
int MAX_COUNT1_RECORDS = 0;
|
||||||
|
|
||||||
|
@Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false)
|
||||||
|
int minCountForDiff = 1;
|
||||||
|
|
||||||
|
@Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false)
|
||||||
|
boolean showItemizedDifferences = false;
|
||||||
|
|
||||||
|
@Argument(fullName="master", shortName="m", doc="Master file: expected results", required=true)
|
||||||
|
File masterFile;
|
||||||
|
|
||||||
|
@Argument(fullName="test", shortName="t", doc="Test file: new results to compare to the master file", required=true)
|
||||||
|
File testFile;
|
||||||
|
|
||||||
|
final DiffEngine diffEngine = new DiffEngine();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void initialize() {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Integer reduceInit() {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Integer reduce(Integer counter, Integer sum) {
|
||||||
|
return counter + sum;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onTraversalDone(Integer sum) {
|
||||||
|
out.printf("Reading master file %s%n", masterFile);
|
||||||
|
DiffElement master = diffEngine.createDiffableFromFile(masterFile);
|
||||||
|
out.printf("Reading test file %s%n", testFile);
|
||||||
|
DiffElement test = diffEngine.createDiffableFromFile(testFile);
|
||||||
|
|
||||||
|
// out.printf("Master diff objects%n");
|
||||||
|
// out.println(master.toString());
|
||||||
|
// out.printf("Test diff objects%n");
|
||||||
|
// out.println(test.toString());
|
||||||
|
|
||||||
|
List<Difference> diffs = diffEngine.diff(master, test);
|
||||||
|
if ( showItemizedDifferences ) {
|
||||||
|
out.printf("Itemized results%n");
|
||||||
|
for ( Difference diff : diffs )
|
||||||
|
out.printf("DIFF: %s%n", diff.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
DiffEngine.SummaryReportParams params = new DiffEngine.SummaryReportParams(out, MAX_RECORDS, MAX_COUNT1_RECORDS, minCountForDiff);
|
||||||
|
diffEngine.reportSummarizedDifferences(diffs, params);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import org.broadinstitute.sting.utils.Utils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 12:55 PM
|
||||||
|
*
|
||||||
|
* An interface that must be implemented to allow us to calculate differences
|
||||||
|
* between structured objects
|
||||||
|
*/
|
||||||
|
public class DiffValue {
|
||||||
|
private DiffElement binding = null;
|
||||||
|
final private Object value;
|
||||||
|
|
||||||
|
public DiffValue(Object value) {
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffValue(DiffElement binding, Object value) {
|
||||||
|
this.binding = binding;
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffValue(DiffValue parent, Object value) {
|
||||||
|
this(parent.getBinding(), value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffValue(String name, DiffElement parent, Object value) {
|
||||||
|
this.binding = new DiffElement(name, parent, this);
|
||||||
|
this.value = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffValue(String name, DiffValue parent, Object value) {
|
||||||
|
this(name, parent.getBinding(), value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public DiffElement getBinding() {
|
||||||
|
return binding;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void setBinding(DiffElement binding) {
|
||||||
|
this.binding = binding;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Object getValue() {
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
return getValue().toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString(int offset) {
|
||||||
|
return toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toOneLineString() {
|
||||||
|
return getValue().toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isAtomic() { return true; }
|
||||||
|
public boolean isCompound() { return ! isAtomic(); }
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import com.google.java.contract.Ensures;
|
||||||
|
import com.google.java.contract.Requires;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 1:09 PM
|
||||||
|
*
|
||||||
|
* Interface for readers creating diffable objects from a file
|
||||||
|
*/
|
||||||
|
public interface DiffableReader {
|
||||||
|
@Ensures("result != null")
|
||||||
|
public String getName();
|
||||||
|
|
||||||
|
@Ensures("result != null")
|
||||||
|
@Requires("file != null")
|
||||||
|
public DiffElement readFromFile(File file);
|
||||||
|
|
||||||
|
@Requires("file != null")
|
||||||
|
public boolean canRead(File file);
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 12:53 PM
|
||||||
|
*
|
||||||
|
* Represents a specific difference between two specific DiffElements
|
||||||
|
*/
|
||||||
|
public class Difference {
|
||||||
|
DiffElement master, test;
|
||||||
|
|
||||||
|
public Difference(DiffElement master, DiffElement test) {
|
||||||
|
if ( master == null && test == null ) throw new IllegalArgumentException("Master and test both cannot be null");
|
||||||
|
this.master = master;
|
||||||
|
this.test = test;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
return String.format("%s:%s!=%s",
|
||||||
|
getFullyQualifiedName(),
|
||||||
|
getOneLineString(master),
|
||||||
|
getOneLineString(test));
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getFullyQualifiedName() {
|
||||||
|
return (master == null ? test : master).fullyQualifiedName();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String getOneLineString(DiffElement elt) {
|
||||||
|
return elt == null ? "MISSING" : elt.getValue().toOneLineString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2011, The Broad Institute
|
||||||
|
*
|
||||||
|
* Permission is hereby granted, free of charge, to any person
|
||||||
|
* obtaining a copy of this software and associated documentation
|
||||||
|
* files (the "Software"), to deal in the Software without
|
||||||
|
* restriction, including without limitation the rights to use,
|
||||||
|
* copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
* copies of the Software, and to permit persons to whom the
|
||||||
|
* Software is furnished to do so, subject to the following
|
||||||
|
* conditions:
|
||||||
|
*
|
||||||
|
* The above copyright notice and this permission notice shall be
|
||||||
|
* included in all copies or substantial portions of the Software.
|
||||||
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
|
||||||
|
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
||||||
|
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
|
||||||
|
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
||||||
|
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
* OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.broadinstitute.sting.gatk.walkers.diffengine;
|
||||||
|
|
||||||
|
import org.broad.tribble.readers.AsciiLineReader;
|
||||||
|
import org.broad.tribble.readers.LineReader;
|
||||||
|
import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
|
||||||
|
import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
|
||||||
|
import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
|
||||||
|
import org.broadinstitute.sting.utils.variantcontext.Genotype;
|
||||||
|
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.zip.GZIPInputStream;
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by IntelliJ IDEA.
|
||||||
|
* User: depristo
|
||||||
|
* Date: 7/4/11
|
||||||
|
* Time: 1:09 PM
|
||||||
|
*
|
||||||
|
* Class implementing diffnode reader for VCF
|
||||||
|
*/
|
||||||
|
public class VCFDiffableReader implements DiffableReader {
|
||||||
|
@Override
|
||||||
|
public String getName() { return "VCF"; }
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DiffElement readFromFile(File file) {
|
||||||
|
DiffNode root = DiffNode.rooted(file.getName());
|
||||||
|
try {
|
||||||
|
LineReader lineReader = new AsciiLineReader(new FileInputStream(file));
|
||||||
|
VCFCodec vcfCodec = new VCFCodec();
|
||||||
|
VCFHeader header = (VCFHeader)vcfCodec.readHeader(lineReader);
|
||||||
|
|
||||||
|
String line = lineReader.readLine();
|
||||||
|
while ( line != null ) {
|
||||||
|
VariantContext vc = (VariantContext)vcfCodec.decode(line);
|
||||||
|
String name = vc.getChr() + ":" + vc.getStart();
|
||||||
|
DiffNode vcRoot = DiffNode.empty(name, root);
|
||||||
|
|
||||||
|
// add fields
|
||||||
|
vcRoot.add("CHROM", vc.getChr());
|
||||||
|
vcRoot.add("POS", vc.getStart());
|
||||||
|
vcRoot.add("ID", vc.hasID() ? vc.getID() : VCFConstants.MISSING_VALUE_v4);
|
||||||
|
vcRoot.add("REF", vc.getReference());
|
||||||
|
vcRoot.add("ALT", vc.getAlternateAlleles());
|
||||||
|
vcRoot.add("QUAL", vc.hasNegLog10PError() ? vc.getNegLog10PError() * 10 : VCFConstants.MISSING_VALUE_v4);
|
||||||
|
vcRoot.add("FILTER", vc.getFilters());
|
||||||
|
|
||||||
|
// add info fields
|
||||||
|
for (Map.Entry<String, Object> attribute : vc.getAttributes().entrySet()) {
|
||||||
|
if ( ! attribute.getKey().startsWith("_") && ! attribute.getKey().equals(VariantContext.ID_KEY))
|
||||||
|
vcRoot.add(attribute.getKey(), attribute.getValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Genotype g : vc.getGenotypes().values() ) {
|
||||||
|
DiffNode gRoot = DiffNode.empty(g.getSampleName(), vcRoot);
|
||||||
|
gRoot.add("GT", g.getGenotypeString());
|
||||||
|
gRoot.add("GQ", g.hasNegLog10PError() ? g.getNegLog10PError() * 10 : VCFConstants.MISSING_VALUE_v4 );
|
||||||
|
|
||||||
|
for (Map.Entry<String, Object> attribute : g.getAttributes().entrySet()) {
|
||||||
|
if ( ! attribute.getKey().startsWith("_") )
|
||||||
|
gRoot.add(attribute.getKey(), attribute.getValue());
|
||||||
|
}
|
||||||
|
|
||||||
|
vcRoot.add(gRoot);
|
||||||
|
}
|
||||||
|
|
||||||
|
root.add(vcRoot);
|
||||||
|
line = lineReader.readLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
lineReader.close();
|
||||||
|
} catch ( IOException e ) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return root.getBinding();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean canRead(File file) {
|
||||||
|
try {
|
||||||
|
final String VCF4_HEADER = "##fileformat=VCFv4";
|
||||||
|
char[] buff = new char[VCF4_HEADER.length()];
|
||||||
|
new FileReader(file).read(buff, 0, VCF4_HEADER.length());
|
||||||
|
String firstLine = new String(buff);
|
||||||
|
return firstLine.startsWith(VCF4_HEADER);
|
||||||
|
} catch ( IOException e ) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue