zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Making the package public, so there's no dependances from public -> private

This commit is contained in:
Mark DePristo 2011-07-11 23:15:08 -04:00
parent 5e593793af
commit 893cc2e103
9 changed files with 1332 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/BAMDiffableReader.java 100644
View File

@ -0,0 +1,122 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import net.sf.samtools.*;
import net.sf.samtools.util.BlockCompressedInputStream;
import org.broad.tribble.readers.AsciiLineReader;
import org.broad.tribble.readers.LineReader;
import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
import org.broadinstitute.sting.utils.variantcontext.Genotype;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import java.util.zip.GZIPInputStream;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 1:09 PM
*
* Class implementing diffnode reader for VCF
*/
public class BAMDiffableReader implements DiffableReader {
private final static int MAX_RECORDS_TO_READ = 1000;
@Override
public String getName() { return "BAM"; }
@Override
public DiffElement readFromFile(File file) {
final SAMFileReader reader = new SAMFileReader(file, null); // null because we don't want it to look for the index
reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
DiffNode root = DiffNode.rooted(file.getName());
SAMRecordIterator iterator = reader.iterator();
int count = 0;
while ( iterator.hasNext() ) {
if ( count++ > MAX_RECORDS_TO_READ )
break;
final SAMRecord record = iterator.next();
// name is the read name + first of pair
String name = record.getReadName().replace('.', '_');
if ( record.getReadPairedFlag() ) {
name += record.getFirstOfPairFlag() ? "_1" : "_2";
}
DiffNode readRoot = DiffNode.empty(name, root);
// add fields
readRoot.add("NAME", record.getReadName());
readRoot.add("FLAGS", record.getFlags());
readRoot.add("RNAME", record.getReferenceName());
readRoot.add("POS", record.getAlignmentStart());
readRoot.add("MAPQ", record.getMappingQuality());
readRoot.add("CIGAR", record.getCigarString());
readRoot.add("RNEXT", record.getMateReferenceName());
readRoot.add("PNEXT", record.getMateAlignmentStart());
readRoot.add("TLEN", record.getInferredInsertSize());
readRoot.add("SEQ", record.getReadString());
readRoot.add("QUAL", record.getBaseQualityString());
for ( SAMRecord.SAMTagAndValue xt : record.getAttributes() ) {
readRoot.add(xt.tag, xt.value);
}
// add record to root
if ( ! root.hasElement(name) )
// protect ourselves from malformed files
root.add(readRoot);
}
reader.close();
return root.getBinding();
}
@Override
public boolean canRead(File file) {
final byte[] BAM_MAGIC = "BAM\1".getBytes();
final byte[] buffer = new byte[BAM_MAGIC.length];
try {
FileInputStream fstream = new FileInputStream(file);
new BlockCompressedInputStream(fstream).read(buffer,0,BAM_MAGIC.length);
return Arrays.equals(buffer, BAM_MAGIC);
} catch ( IOException e ) {
return false;
} catch ( net.sf.samtools.FileTruncatedException e ) {
return false;
}
}
}

118
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffElement.java 100644
View File

@ -0,0 +1,118 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import com.google.java.contract.*;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 12:55 PM
*
* An interface that must be implemented to allow us to calculate differences
* between structured objects
*/
@Invariant({
"name != null",
"value != null",
"parent != null || name.equals(\"ROOT\")",
"value == null || value.getBinding() == this"})
public class DiffElement {
public final static DiffElement ROOT = new DiffElement();
final private String name;
final private DiffElement parent;
final private DiffValue value;
/**
* For ROOT only
*/
private DiffElement() {
this.name = "ROOT";
this.parent = null;
this.value = new DiffValue(this, "ROOT");
}
@Requires({"name != null", "parent != null", "value != null"})
public DiffElement(String name, DiffElement parent, DiffValue value) {
if ( name.equals("ROOT") ) throw new IllegalArgumentException("Cannot use reserved name ROOT");
this.name = name;
this.parent = parent;
this.value = value;
this.value.setBinding(this);
}
@Ensures({"result != null"})
public String getName() {
return name;
}
public DiffElement getParent() {
return parent;
}
@Ensures({"result != null"})
public DiffValue getValue() {
return value;
}
public boolean isRoot() { return this == ROOT; }
@Ensures({"result != null"})
@Override
public String toString() {
return getName() + "=" + getValue().toString();
}
public String toString(int offset) {
return (offset > 0 ? Utils.dupString(' ', offset) : 0) + getName() + "=" + getValue().toString(offset);
}
@Ensures({"result != null"})
public final String fullyQualifiedName() {
if ( isRoot() )
return "";
else if ( parent.isRoot() )
return name;
else
return parent.fullyQualifiedName() + "." + name;
}
@Ensures({"result != null"})
public String toOneLineString() {
return getName() + "=" + getValue().toOneLineString();
}
@Ensures({"result != null"})
public DiffNode getValueAsNode() {
if ( getValue().isCompound() )
return (DiffNode)getValue();
else
throw new ReviewedStingException("Illegal request conversion of a DiffValue into a DiffNode: " + this);
}
}

423
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java 100644
View File

@ -0,0 +1,423 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import com.google.java.contract.Requires;
import org.apache.log4j.Logger;
import org.broadinstitute.sting.gatk.report.GATKReport;
import org.broadinstitute.sting.gatk.report.GATKReportTable;
import org.broadinstitute.sting.gatk.walkers.varianteval.stratifications.VariantStratifier;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import org.broadinstitute.sting.utils.exceptions.UserException;
import java.io.File;
import java.io.PrintStream;
import java.util.*;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 12:51 PM
* A generic engine for comparing tree-structured objects
*/
public class DiffEngine {
final protected static Logger logger = Logger.getLogger(DiffEngine.class);
private final Map<String, DiffableReader> readers = new HashMap<String, DiffableReader>();
public DiffEngine() {
loadDiffableReaders();
}
// --------------------------------------------------------------------------------
//
// difference calculation
//
// --------------------------------------------------------------------------------
public List<Difference> diff(DiffElement master, DiffElement test) {
DiffValue masterValue = master.getValue();
DiffValue testValue = test.getValue();
if ( masterValue.isCompound() && masterValue.isCompound() ) {
return diff(master.getValueAsNode(), test.getValueAsNode());
} else if ( masterValue.isAtomic() && testValue.isAtomic() ) {
return diff(masterValue, testValue);
} else {
// structural difference in types. one is node, other is leaf
return Arrays.asList(new Difference(master, test));
}
}
public List<Difference> diff(DiffNode master, DiffNode test) {
Set<String> allNames = new HashSet<String>(master.getElementNames());
allNames.addAll(test.getElementNames());
List<Difference> diffs = new ArrayList<Difference>();
for ( String name : allNames ) {
DiffElement masterElt = master.getElement(name);
DiffElement testElt = test.getElement(name);
if ( masterElt == null && testElt == null ) {
throw new ReviewedStingException("BUG: unexceptedly got two null elements for field: " + name);
} else if ( masterElt == null || testElt == null ) { // if either is null, we are missing a value
// todo -- should one of these be a special MISSING item?
diffs.add(new Difference(masterElt, testElt));
} else {
diffs.addAll(diff(masterElt, testElt));
}
}
return diffs;
}
public List<Difference> diff(DiffValue master, DiffValue test) {
if ( master.getValue().equals(test.getValue()) ) {
return Collections.emptyList();
} else {
return Arrays.asList(new Difference(master.getBinding(), test.getBinding()));
}
}
// --------------------------------------------------------------------------------
//
// Summarizing differences
//
// --------------------------------------------------------------------------------
/**
* Emits a summary of the diffs to out. Suppose you have the following three differences:
*
* A.X.Z:1!=2
* A.Y.Z:3!=4
* B.X.Z:5!=6
*
* The above is the itemized list of the differences. The summary looks for common differences
* in the name hierarchy, counts those shared elements, and emits the differences that occur
* in order of decreasing counts.
*
* So, in the above example, what are the shared elements?
*
* A.X.Z and B.X.Z share X.Z, so there's a *.X.Z with count 2
* A.X.Z, A.Y.Z, and B.X.Z all share *.*.Z, with count 3
* Each of A.X.Z, A.Y.Z, and B.X.Z are individually unique, with count 1
*
* So we would emit the following summary:
*
* *.*.Z: 3
* *.X.Z: 2
* A.X.Z: 1 [specific difference: 1!=2]
* A.Y.Z: 1 [specific difference: 3!=4]
* B.X.Z: 1 [specific difference: 5!=6]
*
* The algorithm to accomplish this calculation is relatively simple. Start with all of the
* concrete differences. For each pair of differences A1.A2....AN and B1.B2....BN:
*
* find the longest common subsequence Si.Si+1...SN where Ai = Bi = Si
* If i == 0, then there's no shared substructure
* If i > 0, then generate the summarized value X = *.*...Si.Si+1...SN
* if X is a known summary, increment it's count, otherwise set its count to 1
*
* Not that only pairs of the same length are considered as potentially equivalent
*
* @param params determines how we display the items
* @param diffs
*/
public void reportSummarizedDifferences(List<Difference> diffs, SummaryReportParams params ) {
printSummaryReport(summarizeDifferences(diffs), params );
}
public List<SummarizedDifference> summarizeDifferences(List<Difference> diffs) {
List<String[]> diffPaths = new ArrayList<String[]>(diffs.size());
for ( Difference diff1 : diffs ) {
diffPaths.add(diffNameToPath(diff1.getFullyQualifiedName()));
}
return summarizedDifferencesOfPaths(diffPaths);
}
final protected static String[] diffNameToPath(String diffName) {
return diffName.split("\\.");
}
protected List<SummarizedDifference> summarizedDifferencesOfPaths(List<String[]> diffPaths) {
Map<String, SummarizedDifference> summaries = new HashMap<String, SummarizedDifference>();
// create the initial set of differences
for ( int i = 0; i < diffPaths.size(); i++ ) {
for ( int j = 0; j <= i; j++ ) {
String[] diffPath1 = diffPaths.get(i);
String[] diffPath2 = diffPaths.get(j);
if ( diffPath1.length == diffPath2.length ) {
int lcp = longestCommonPostfix(diffPath1, diffPath2);
String path = lcp > 0 ? summarizedPath(diffPath2, lcp) : Utils.join(".", diffPath2);
addSummary(summaries, path, true);
}
}
}
// count differences
for ( String[] diffPath : diffPaths ) {
for ( SummarizedDifference sumDiff : summaries.values() ) {
if ( sumDiff.matches(diffPath) )
addSummary(summaries, sumDiff.getPath(), false);
}
}
List<SummarizedDifference> sortedSummaries = new ArrayList<SummarizedDifference>(summaries.values());
Collections.sort(sortedSummaries);
return sortedSummaries;
}
private static void addSummary(Map<String, SummarizedDifference> summaries, String path, boolean onlyCatalog) {
if ( summaries.containsKey(path) ) {
if ( ! onlyCatalog )
summaries.get(path).incCount();
} else {
SummarizedDifference sumDiff = new SummarizedDifference(path);
summaries.put(sumDiff.getPath(), sumDiff);
}
}
protected void printSummaryReport(List<SummarizedDifference> sortedSummaries, SummaryReportParams params ) {
GATKReport report = new GATKReport();
final String tableName = "diffences";
report.addTable(tableName, "Summarized differences between the master and test files.\nSee http://www.broadinstitute.org/gsa/wiki/index.php/DiffObjectsWalker_and_SummarizedDifferences for more information");
GATKReportTable table = report.getTable(tableName);
table.addPrimaryKey("Difference", true);
table.addColumn("NumberOfOccurrences", 0);
int count = 0, count1 = 0;
for ( SummarizedDifference diff : sortedSummaries ) {
if ( diff.getCount() < params.minSumDiffToShow )
// in order, so break as soon as the count is too low
break;
if ( params.maxItemsToDisplay != 0 && count++ > params.maxItemsToDisplay )
break;
if ( diff.getCount() == 1 ) {
count1++;
if ( params.maxCountOneItems != 0 && count1 > params.maxCountOneItems )
break;
}
table.set(diff.getPath(), "NumberOfOccurrences", diff.getCount());
}
table.write(params.out);
}
protected static int longestCommonPostfix(String[] diffPath1, String[] diffPath2) {
int i = 0;
for ( ; i < diffPath1.length; i++ ) {
int j = diffPath1.length - i - 1;
if ( ! diffPath1[j].equals(diffPath2[j]) )
break;
}
return i;
}
/**
* parts is [A B C D]
* commonPostfixLength: how many parts are shared at the end, suppose its 2
* We want to create a string *.*.C.D
*
* @param parts
* @param commonPostfixLength
* @return
*/
protected static String summarizedPath(String[] parts, int commonPostfixLength) {
int stop = parts.length - commonPostfixLength;
if ( stop > 0 ) parts = parts.clone();
for ( int i = 0; i < stop; i++ ) {
parts[i] = "*";
}
return Utils.join(".", parts);
}
/**
* TODO -- all of the algorithms above should use SummarizedDifference instead
* TODO -- of some SummarizedDifferences and some low-level String[]
*/
public static class SummarizedDifference implements Comparable<SummarizedDifference> {
final String path; // X.Y.Z
final String[] parts;
int count = 0;
public SummarizedDifference(String path) {
this.path = path;
this.parts = diffNameToPath(path);
}
public void incCount() { count++; }
public int getCount() {
return count;
}
/**
* The fully qualified path object A.B.C etc
* @return
*/
public String getPath() {
return path;
}
/**
* @return the length of the parts of this summary
*/
public int length() {
return this.parts.length;
}
/**
* Returns true if the string parts matches this summary. Matches are
* must be equal() everywhere where this summary isn't *.
* @param otherParts
* @return
*/
public boolean matches(String[] otherParts) {
if ( otherParts.length != length() )
return false;
// TODO optimization: can start at right most non-star element
for ( int i = 0; i < length(); i++ ) {
String part = parts[i];
if ( ! part.equals("*") && ! part.equals(otherParts[i]) )
return false;
}
return true;
}
@Override
public String toString() {
return String.format("%s:%d", getPath(), getCount());
}
@Override
public int compareTo(SummarizedDifference other) {
// sort first highest to lowest count, then by lowest to highest path
int countCmp = Integer.valueOf(count).compareTo(other.count);
return countCmp != 0 ? -1 * countCmp : path.compareTo(other.path);
}
}
// --------------------------------------------------------------------------------
//
// plugin manager
//
// --------------------------------------------------------------------------------
public void loadDiffableReaders() {
List<Class<? extends DiffableReader>> drClasses = new PluginManager<DiffableReader>( DiffableReader.class ).getPlugins();
logger.info("Loading diffable modules:");
for (Class<? extends DiffableReader> drClass : drClasses ) {
logger.info("\t" + drClass.getSimpleName());
try {
DiffableReader dr = drClass.newInstance();
readers.put(dr.getName(), dr);
} catch (InstantiationException e) {
throw new ReviewedStingException("Unable to instantiate module '" + drClass.getSimpleName() + "'");
} catch (IllegalAccessException e) {
throw new ReviewedStingException("Illegal access error when trying to instantiate '" + drClass.getSimpleName() + "'");
}
}
}
protected Map<String, DiffableReader> getReaders() {
return readers;
}
protected DiffableReader getReader(String name) {
return readers.get(name);
}
/**
* Returns a reader appropriate for this file, or null if no such reader exists
* @param file
* @return
*/
public DiffableReader findReaderForFile(File file) {
for ( DiffableReader reader : readers.values() )
if (reader.canRead(file) )
return reader;
return null;
}
/**
* Returns true if reader appropriate for this file, or false if no such reader exists
* @param file
* @return
*/
public boolean canRead(File file) {
return findReaderForFile(file) != null;
}
public DiffElement createDiffableFromFile(File file) {
DiffableReader reader = findReaderForFile(file);
if ( reader == null )
throw new UserException("Unsupported file type: " + file);
else
return reader.readFromFile(file);
}
public static boolean simpleDiffFiles(File masterFile, File testFile, DiffEngine.SummaryReportParams params) {
DiffEngine diffEngine = new DiffEngine();
if ( diffEngine.canRead(masterFile) && diffEngine.canRead(testFile) ) {
DiffElement master = diffEngine.createDiffableFromFile(masterFile);
DiffElement test = diffEngine.createDiffableFromFile(testFile);
List<Difference> diffs = diffEngine.diff(master, test);
diffEngine.reportSummarizedDifferences(diffs, params);
return true;
} else {
return false;
}
}
public static class SummaryReportParams {
PrintStream out = System.out;
int maxItemsToDisplay = 0;
int maxCountOneItems = 0;
int minSumDiffToShow = 0;
public SummaryReportParams(PrintStream out, int maxItemsToDisplay, int maxCountOneItems, int minSumDiffToShow) {
this.out = out;
this.maxItemsToDisplay = maxItemsToDisplay;
this.maxCountOneItems = maxCountOneItems;
this.minSumDiffToShow = minSumDiffToShow;
}
}
}

239
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffNode.java 100644
View File

@ -0,0 +1,239 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import com.google.java.contract.Requires;
import org.broadinstitute.sting.utils.Utils;
import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
import java.util.*;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 12:55 PM
*
* An interface that must be implemented to allow us to calculate differences
* between structured objects
*/
public class DiffNode extends DiffValue {
private Map<String, DiffElement> getElementMap() {
return (Map<String, DiffElement>)super.getValue();
}
private static Map<String, DiffElement> emptyElements() { return new HashMap<String, DiffElement>(); }
private DiffNode(Map<String, DiffElement> elements) {
super(elements);
}
private DiffNode(DiffElement binding, Map<String, DiffElement> elements) {
super(binding, elements);
}
// ---------------------------------------------------------------------------
//
// constructors
//
// ---------------------------------------------------------------------------
public static DiffNode rooted(String name) {
return empty(name, DiffElement.ROOT);
}
public static DiffNode empty(String name, DiffElement parent) {
DiffNode df = new DiffNode(emptyElements());
DiffElement elt = new DiffElement(name, parent, df);
df.setBinding(elt);
return df;
}
public static DiffNode empty(String name, DiffValue parent) {
return empty(name, parent.getBinding());
}
// ---------------------------------------------------------------------------
//
// accessors
//
// ---------------------------------------------------------------------------
@Override
public boolean isAtomic() { return false; }
public Collection<String> getElementNames() {
return getElementMap().keySet();
}
public Collection<DiffElement> getElements() {
return getElementMap().values();
}
private Collection<DiffElement> getElements(boolean atomicOnly) {
List<DiffElement> elts = new ArrayList<DiffElement>();
for ( DiffElement elt : getElements() )
if ( (atomicOnly && elt.getValue().isAtomic()) || (! atomicOnly && elt.getValue().isCompound()))
elts.add(elt);
return elts;
}
public Collection<DiffElement> getAtomicElements() {
return getElements(true);
}
public Collection<DiffElement> getCompoundElements() {
return getElements(false);
}
public DiffElement getElement(String name) {
for ( DiffElement elt : getElements() )
if ( elt.getName().equals(name) )
return elt;
return null;
}
/**
* Returns true if name is bound in this node
* @param name
* @return
*/
public boolean hasElement(String name) {
return getElement(name) != null;
}
// ---------------------------------------------------------------------------
//
// add
//
// ---------------------------------------------------------------------------
@Requires("elt != null")
public void add(DiffElement elt) {
if ( getElementMap().containsKey(elt.getName()) )
throw new IllegalArgumentException("Attempting to rebind already existing binding: " + elt + " node=" + this);
getElementMap().put(elt.getName(), elt);
}
@Requires("elt != null")
public void add(DiffValue elt) {
add(elt.getBinding());
}
@Requires("elts != null")
public void add(Collection<DiffElement> elts) {
for ( DiffElement e : elts )
add(e);
}
public void add(String name, Object value) {
add(new DiffElement(name, this.getBinding(), new DiffValue(value)));
}
// ---------------------------------------------------------------------------
//
// toString
//
// ---------------------------------------------------------------------------
@Override
public String toString() {
return toString(0);
}
@Override
public String toString(int offset) {
String off = offset > 0 ? Utils.dupString(' ', offset) : "";
StringBuilder b = new StringBuilder();
b.append("(").append("\n");
Collection<DiffElement> atomicElts = getAtomicElements();
for ( DiffElement elt : atomicElts ) {
b.append(elt.toString(offset + 2)).append('\n');
}
for ( DiffElement elt : getCompoundElements() ) {
b.append(elt.toString(offset + 4)).append('\n');
}
b.append(off).append(")").append("\n");
return b.toString();
}
@Override
public String toOneLineString() {
StringBuilder b = new StringBuilder();
b.append('(');
List<String> parts = new ArrayList<String>();
for ( DiffElement elt : getElements() )
parts.add(elt.toOneLineString());
b.append(Utils.join(" ", parts));
b.append(')');
return b.toString();
}
// --------------------------------------------------------------------------------
//
// fromString and toOneLineString
//
// --------------------------------------------------------------------------------
public static DiffElement fromString(String tree) {
return fromString(tree, DiffElement.ROOT);
}
/**
* Doesn't support full tree structure parsing
* @param tree
* @param parent
* @return
*/
private static DiffElement fromString(String tree, DiffElement parent) {
// X=(A=A B=B C=(D=D))
String[] parts = tree.split("=", 2);
if ( parts.length != 2 )
throw new ReviewedStingException("Unexpected tree structure: " + tree + " parts=" + parts);
String name = parts[0];
String value = parts[1];
if ( value.length() == 0 )
throw new ReviewedStingException("Illegal tree structure: " + value + " at " + tree);
if ( value.charAt(0) == '(' ) {
if ( ! value.endsWith(")") )
throw new ReviewedStingException("Illegal tree structure. Missing ): " + value + " at " + tree);
String subtree = value.substring(1, value.length()-1);
DiffNode rec = DiffNode.empty(name, parent);
String[] subParts = subtree.split(" ");
for ( String subPart : subParts ) {
rec.add(fromString(subPart, rec.getBinding()));
}
return rec.getBinding();
} else {
return new DiffValue(name, parent, value).getBinding();
}
}
}

113
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java 100644
View File

@ -0,0 +1,113 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import org.apache.xmlbeans.impl.tool.Diff;
import org.broadinstitute.sting.commandline.Argument;
import org.broadinstitute.sting.commandline.Output;
import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
import org.broadinstitute.sting.gatk.walkers.Requires;
import org.broadinstitute.sting.gatk.walkers.RodWalker;
import java.io.File;
import java.io.PrintStream;
import java.util.List;
/**
* Compares two record-oriented files, itemizing specific difference between equivalent
* records in the two files. Reports both itemized and summarized differences.
* @author Mark DePristo
* @version 0.1
*/
@Requires(value={})
public class DiffObjectsWalker extends RodWalker<Integer, Integer> {
@Output(doc="File to which results should be written",required=true)
protected PrintStream out;
@Argument(fullName="maxRecords", shortName="M", doc="Max. number of records to process", required=false)
int MAX_RECORDS = 0;
@Argument(fullName="maxCount1Records", shortName="M1", doc="Max. number of records occuring exactly once in the file to process", required=false)
int MAX_COUNT1_RECORDS = 0;
@Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false)
int minCountForDiff = 1;
@Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false)
boolean showItemizedDifferences = false;
@Argument(fullName="master", shortName="m", doc="Master file: expected results", required=true)
File masterFile;
@Argument(fullName="test", shortName="t", doc="Test file: new results to compare to the master file", required=true)
File testFile;
final DiffEngine diffEngine = new DiffEngine();
@Override
public void initialize() {
}
@Override
public Integer map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext context) {
return 0;
}
@Override
public Integer reduceInit() {
return 0;
}
@Override
public Integer reduce(Integer counter, Integer sum) {
return counter + sum;
}
@Override
public void onTraversalDone(Integer sum) {
out.printf("Reading master file %s%n", masterFile);
DiffElement master = diffEngine.createDiffableFromFile(masterFile);
out.printf("Reading test file %s%n", testFile);
DiffElement test = diffEngine.createDiffableFromFile(testFile);
// out.printf("Master diff objects%n");
// out.println(master.toString());
// out.printf("Test diff objects%n");
// out.println(test.toString());
List<Difference> diffs = diffEngine.diff(master, test);
if ( showItemizedDifferences ) {
out.printf("Itemized results%n");
for ( Difference diff : diffs )
out.printf("DIFF: %s%n", diff.toString());
}
DiffEngine.SummaryReportParams params = new DiffEngine.SummaryReportParams(out, MAX_RECORDS, MAX_COUNT1_RECORDS, minCountForDiff);
diffEngine.reportSummarizedDifferences(diffs, params);
}
}

90
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffValue.java 100644
View File

@ -0,0 +1,90 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import org.broadinstitute.sting.utils.Utils;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 12:55 PM
*
* An interface that must be implemented to allow us to calculate differences
* between structured objects
*/
public class DiffValue {
private DiffElement binding = null;
final private Object value;
public DiffValue(Object value) {
this.value = value;
}
public DiffValue(DiffElement binding, Object value) {
this.binding = binding;
this.value = value;
}
public DiffValue(DiffValue parent, Object value) {
this(parent.getBinding(), value);
}
public DiffValue(String name, DiffElement parent, Object value) {
this.binding = new DiffElement(name, parent, this);
this.value = value;
}
public DiffValue(String name, DiffValue parent, Object value) {
this(name, parent.getBinding(), value);
}
public DiffElement getBinding() {
return binding;
}
protected void setBinding(DiffElement binding) {
this.binding = binding;
}
public Object getValue() {
return value;
}
public String toString() {
return getValue().toString();
}
public String toString(int offset) {
return toString();
}
public String toOneLineString() {
return getValue().toString();
}
public boolean isAtomic() { return true; }
public boolean isCompound() { return ! isAtomic(); }
}

50
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffableReader.java 100644
View File

@ -0,0 +1,50 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import com.google.java.contract.Ensures;
import com.google.java.contract.Requires;
import java.io.File;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 1:09 PM
*
* Interface for readers creating diffable objects from a file
*/
public interface DiffableReader {
@Ensures("result != null")
public String getName();
@Ensures("result != null")
@Requires("file != null")
public DiffElement readFromFile(File file);
@Requires("file != null")
public boolean canRead(File file);
}

58
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/Difference.java 100644
View File

@ -0,0 +1,58 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 12:53 PM
*
* Represents a specific difference between two specific DiffElements
*/
public class Difference {
DiffElement master, test;
public Difference(DiffElement master, DiffElement test) {
if ( master == null && test == null ) throw new IllegalArgumentException("Master and test both cannot be null");
this.master = master;
this.test = test;
}
public String toString() {
return String.format("%s:%s!=%s",
getFullyQualifiedName(),
getOneLineString(master),
getOneLineString(test));
}
public String getFullyQualifiedName() {
return (master == null ? test : master).fullyQualifiedName();
}
private static String getOneLineString(DiffElement elt) {
return elt == null ? "MISSING" : elt.getValue().toOneLineString();
}
}

119
public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/VCFDiffableReader.java 100644
View File

@ -0,0 +1,119 @@
/*
* Copyright (c) 2011, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.gatk.walkers.diffengine;
import org.broad.tribble.readers.AsciiLineReader;
import org.broad.tribble.readers.LineReader;
import org.broadinstitute.sting.utils.codecs.vcf.VCFCodec;
import org.broadinstitute.sting.utils.codecs.vcf.VCFConstants;
import org.broadinstitute.sting.utils.codecs.vcf.VCFHeader;
import org.broadinstitute.sting.utils.variantcontext.Genotype;
import org.broadinstitute.sting.utils.variantcontext.VariantContext;
import java.io.*;
import java.util.Arrays;
import java.util.Map;
import java.util.zip.GZIPInputStream;
/**
* Created by IntelliJ IDEA.
* User: depristo
* Date: 7/4/11
* Time: 1:09 PM
*
* Class implementing diffnode reader for VCF
*/
public class VCFDiffableReader implements DiffableReader {
@Override
public String getName() { return "VCF"; }
@Override
public DiffElement readFromFile(File file) {
DiffNode root = DiffNode.rooted(file.getName());
try {
LineReader lineReader = new AsciiLineReader(new FileInputStream(file));
VCFCodec vcfCodec = new VCFCodec();
VCFHeader header = (VCFHeader)vcfCodec.readHeader(lineReader);
String line = lineReader.readLine();
while ( line != null ) {
VariantContext vc = (VariantContext)vcfCodec.decode(line);
String name = vc.getChr() + ":" + vc.getStart();
DiffNode vcRoot = DiffNode.empty(name, root);
// add fields
vcRoot.add("CHROM", vc.getChr());
vcRoot.add("POS", vc.getStart());
vcRoot.add("ID", vc.hasID() ? vc.getID() : VCFConstants.MISSING_VALUE_v4);
vcRoot.add("REF", vc.getReference());
vcRoot.add("ALT", vc.getAlternateAlleles());
vcRoot.add("QUAL", vc.hasNegLog10PError() ? vc.getNegLog10PError() * 10 : VCFConstants.MISSING_VALUE_v4);
vcRoot.add("FILTER", vc.getFilters());
// add info fields
for (Map.Entry<String, Object> attribute : vc.getAttributes().entrySet()) {
if ( ! attribute.getKey().startsWith("_") && ! attribute.getKey().equals(VariantContext.ID_KEY))
vcRoot.add(attribute.getKey(), attribute.getValue());
}
for (Genotype g : vc.getGenotypes().values() ) {
DiffNode gRoot = DiffNode.empty(g.getSampleName(), vcRoot);
gRoot.add("GT", g.getGenotypeString());
gRoot.add("GQ", g.hasNegLog10PError() ? g.getNegLog10PError() * 10 : VCFConstants.MISSING_VALUE_v4 );
for (Map.Entry<String, Object> attribute : g.getAttributes().entrySet()) {
if ( ! attribute.getKey().startsWith("_") )
gRoot.add(attribute.getKey(), attribute.getValue());
}
vcRoot.add(gRoot);
}
root.add(vcRoot);
line = lineReader.readLine();
}
lineReader.close();
} catch ( IOException e ) {
return null;
}
return root.getBinding();
}
@Override
public boolean canRead(File file) {
try {
final String VCF4_HEADER = "##fileformat=VCFv4";
char[] buff = new char[VCF4_HEADER.length()];
new FileReader(file).read(buff, 0, VCF4_HEADER.length());
String firstLine = new String(buff);
return firstLine.startsWith(VCF4_HEADER);
} catch ( IOException e ) {
return false;
}
}
}