From 677bea0abdb1010da084818a0b8903f4e6ea1dbf Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 5 Dec 2011 23:04:33 -0500 Subject: [PATCH] Right aligning GATKReport numeric columns and updated MD5s in tests. PreQC parses file with spaces in sample names by using tabs only. PostQC allows passing the file names for the evals so that flanks can be evaled. BaseTest's network temp dir now adds the user name to the path so files aren't created in the root. HybridSelectionPipeline: - Updated to latest versions of reference data. - Refactored Picard parsing code replacing YAML. --- .../sting/gatk/report/GATKReportColumn.java | 44 +++++-- .../report/GATKReportColumnFormat.java} | 50 ++++---- .../sting/gatk/report/GATKReportTable.java | 11 +- .../diffengine/GATKReportDiffableReader.java | 4 +- .../sting/pipeline/Pipeline.java | 62 ---------- .../sting/pipeline/PipelineProject.java | 115 ------------------ .../sting/utils/io/IOUtils.java | 23 ++++ .../utils/yaml/FieldOrderComparator.java | 52 -------- .../utils/yaml/StingYamlRepresenter.java | 88 -------------- .../sting/utils/yaml/YamlUtils.java | 107 ---------------- .../org/broadinstitute/sting/BaseTest.java | 3 +- .../sting/gatk/report/GATKReportUnitTest.java | 28 +++++ .../DiffObjectsIntegrationTest.java | 4 +- .../VariantEvalIntegrationTest.java | 44 +++---- .../VCFStreamingIntegrationTest.java | 2 +- .../sting/pipeline/PipelineUnitTest.java | 88 -------------- 16 files changed, 143 insertions(+), 582 deletions(-) rename public/java/src/org/broadinstitute/sting/{pipeline/PipelineSample.java => gatk/report/GATKReportColumnFormat.java} (55%) delete mode 100644 public/java/src/org/broadinstitute/sting/pipeline/Pipeline.java delete mode 100644 public/java/src/org/broadinstitute/sting/pipeline/PipelineProject.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/yaml/FieldOrderComparator.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/yaml/StingYamlRepresenter.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/yaml/YamlUtils.java delete mode 100644 public/java/test/org/broadinstitute/sting/pipeline/PipelineUnitTest.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumn.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumn.java index 6452c7b2b..5a6490afe 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumn.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumn.java @@ -1,6 +1,8 @@ package org.broadinstitute.sting.gatk.report; -import java.util.TreeMap; +import org.apache.commons.lang.math.NumberUtils; + +import java.util.*; /** * Holds values for a column in a GATK report table @@ -16,12 +18,9 @@ public class GATKReportColumn extends TreeMap { * * @param columnName the name of the column * @param defaultValue the default value of the column - * @param display if true, the column will be displayed in the final output + * @param display if true, the column will be displayed in the final output + * @param format format string */ - public GATKReportColumn(String columnName, Object defaultValue, boolean display) { - this(columnName, defaultValue, display, null); - } - public GATKReportColumn(String columnName, Object defaultValue, boolean display, String format) { this.columnName = columnName; this.defaultValue = defaultValue; @@ -77,22 +76,47 @@ public class GATKReportColumn extends TreeMap { /** * Get the display width for this column. This allows the entire column to be displayed with the appropriate, fixed width. - * @return the width of this column + * @return the format string for this column */ - public int getColumnWidth() { + public GATKReportColumnFormat getColumnFormat() { int maxWidth = columnName.length(); + GATKReportColumnFormat.Alignment alignment = GATKReportColumnFormat.Alignment.RIGHT; for (Object obj : this.values()) { if (obj != null) { - int width = formatValue(obj).length(); + String formatted = formatValue(obj); + int width = formatted.length(); if (width > maxWidth) { maxWidth = width; } + + if (alignment == GATKReportColumnFormat.Alignment.RIGHT) { + if (!isRightAlign(formatted)) { + alignment = GATKReportColumnFormat.Alignment.LEFT; + } + } } } - return maxWidth; + return new GATKReportColumnFormat(maxWidth, alignment); + } + + private static final Collection RIGHT_ALIGN_STRINGS = Arrays.asList( + "null", + "NA", + String.valueOf(Double.POSITIVE_INFINITY), + String.valueOf(Double.NEGATIVE_INFINITY), + String.valueOf(Double.NaN)); + + /** + * Check if the value can be right aligned. Does not trim the values before checking if numeric since it assumes + * the spaces mean that the value is already padded. + * @param value to check + * @return true if the value is a right alignable + */ + protected static boolean isRightAlign(String value) { + return value == null || RIGHT_ALIGN_STRINGS.contains(value) || NumberUtils.isNumber(value); } /** diff --git a/public/java/src/org/broadinstitute/sting/pipeline/PipelineSample.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumnFormat.java similarity index 55% rename from public/java/src/org/broadinstitute/sting/pipeline/PipelineSample.java rename to public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumnFormat.java index 7cd25fed5..6d19a83aa 100644 --- a/public/java/src/org/broadinstitute/sting/pipeline/PipelineSample.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportColumnFormat.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, The Broad Institute + * Copyright (c) 2011, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -22,41 +22,41 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -package org.broadinstitute.sting.pipeline; - -import java.io.File; -import java.util.Map; -import java.util.TreeMap; +package org.broadinstitute.sting.gatk.report; /** - * Java bean defining a sample for a pipeline. + * Column width and left/right alignment. */ -public class PipelineSample { - private String id; - private Map bamFiles = new TreeMap(); - private Map tags = new TreeMap(); +public class GATKReportColumnFormat { + public static enum Alignment { LEFT, RIGHT } + public int width; + public Alignment alignment; - public String getId() { - return id; + public GATKReportColumnFormat(int width, Alignment alignment) { + this.width = width; + this.alignment = alignment; } - public void setId(String id) { - this.id = id; + public int getWidth() { + return width; } - public Map getBamFiles() { - return bamFiles; + public Alignment getAlignment() { + return alignment; } - public void setBamFiles(Map bamFiles) { - this.bamFiles = bamFiles; + public String getNameFormat() { + return "%-" + width + "s"; } - public Map getTags() { - return tags; - } - - public void setTags(Map tags) { - this.tags = tags; + public String getValueFormat() { + switch (alignment) { + case LEFT: + return "%-" + width + "s"; + case RIGHT: + return "%" + width + "s"; + default: + throw new UnsupportedOperationException("Unknown alignment: " + alignment); + } } } diff --git a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java index 95c2a14fc..b72b20e0b 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java +++ b/public/java/src/org/broadinstitute/sting/gatk/report/GATKReportTable.java @@ -608,12 +608,9 @@ public class GATKReportTable { */ public void write(PrintStream out) { // Get the column widths for everything - HashMap columnWidths = new HashMap(); + HashMap columnFormats = new HashMap(); for (String columnName : columns.keySet()) { - int width = columns.get(columnName).getColumnWidth(); - String format = "%-" + String.valueOf(width) + "s"; - - columnWidths.put(columnName, format); + columnFormats.put(columnName, columns.get(columnName).getColumnFormat()); } String primaryKeyFormat = "%-" + getPrimaryKeyColumnWidth() + "s"; @@ -630,7 +627,7 @@ public class GATKReportTable { for (String columnName : columns.keySet()) { if (columns.get(columnName).isDisplayable()) { if (needsPadding) { out.printf(" "); } - out.printf(columnWidths.get(columnName), columnName); + out.printf(columnFormats.get(columnName).getNameFormat(), columnName); needsPadding = true; } @@ -650,7 +647,7 @@ public class GATKReportTable { if (columns.get(columnName).isDisplayable()) { if (needsPadding) { out.printf(" "); } String value = columns.get(columnName).getStringValue(primaryKey); - out.printf(columnWidths.get(columnName), value); + out.printf(columnFormats.get(columnName).getValueFormat(), value); needsPadding = true; } diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/GATKReportDiffableReader.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/GATKReportDiffableReader.java index ef47ee33c..41b17cc7b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/GATKReportDiffableReader.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/GATKReportDiffableReader.java @@ -31,7 +31,6 @@ import org.broadinstitute.sting.gatk.report.GATKReportTable; import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.util.Map; /** @@ -68,7 +67,8 @@ public class GATKReportDiffableReader implements DiffableReader { for ( GATKReportColumn column : table.getColumns().values() ) { DiffNode columnRoot = DiffNode.empty(column.getColumnName(), tableRoot); - columnRoot.add("Width", column.getColumnWidth()); + columnRoot.add("Width", column.getColumnFormat().getWidth()); + // NOTE: as the values are trimmed during parsing left/right alignment is not currently preserved columnRoot.add("Displayable", column.isDisplayable()); int n = 1; diff --git a/public/java/src/org/broadinstitute/sting/pipeline/Pipeline.java b/public/java/src/org/broadinstitute/sting/pipeline/Pipeline.java deleted file mode 100644 index e0e75c353..000000000 --- a/public/java/src/org/broadinstitute/sting/pipeline/Pipeline.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.pipeline; - -import java.util.ArrayList; -import java.util.List; - -/** - * Java bean for storing a list of samples for a pipeline. - * - * NOTE: This class is used in a very similar way to the classes in - * org.broadinstitute.sting.gatk.datasources.sample. - * - * Both store / load sample information from the file system as YAML. - * - * This package will likely be refactored to share common functionality - * with the other at a future date as requirements coalesce. - * - * - kshakir September 22, 2010 - */ -public class Pipeline { - private PipelineProject project = new PipelineProject(); - private List samples = new ArrayList(); - - public PipelineProject getProject() { - return project; - } - - public void setProject(PipelineProject project) { - this.project = project; - } - - public List getSamples() { - return samples; - } - - public void setSamples(List samples) { - this.samples = samples; - } -} diff --git a/public/java/src/org/broadinstitute/sting/pipeline/PipelineProject.java b/public/java/src/org/broadinstitute/sting/pipeline/PipelineProject.java deleted file mode 100644 index 8d33047bf..000000000 --- a/public/java/src/org/broadinstitute/sting/pipeline/PipelineProject.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.pipeline; - -import java.io.File; -import java.util.Map; -import java.util.TreeMap; - -/** - * Java bean defining the project for a pipeline. - */ -public class PipelineProject { - private String name; - private File referenceFile; - private File intervalList; - private File genotypeDbsnp; - private File evalDbsnp; - private File refseqTable; - private Map tags = new TreeMap(); - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public File getIntervalList() { - return intervalList; - } - - public void setIntervalList(File intervalList) { - this.intervalList = intervalList; - } - - public File getReferenceFile() { - return referenceFile; - } - - public void setReferenceFile(File referenceFile) { - this.referenceFile = referenceFile; - } - - public File getGenotypeDbsnp() { - return genotypeDbsnp; - } - - public void setGenotypeDbsnp(File genotypeDbsnp) { - this.genotypeDbsnp = genotypeDbsnp; - } - - public String getGenotypeDbsnpType() { - return getDbsnpType(genotypeDbsnp); - } - - public File getEvalDbsnp() { - return evalDbsnp; - } - - public void setEvalDbsnp(File evalDbsnp) { - this.evalDbsnp = evalDbsnp; - } - - public String getEvalDbsnpType() { - return getDbsnpType(evalDbsnp); - } - - public File getRefseqTable() { - return refseqTable; - } - - public void setRefseqTable(File refseqTable) { - this.refseqTable = refseqTable; - } - - public Map getTags() { - return tags; - } - - public void setTags(Map tags) { - this.tags = tags; - } - - private String getDbsnpType(File file) { - if (file == null) - return null; - else if (file.getName().toLowerCase().endsWith(".vcf")) - return "vcf"; - else - return "dbsnp"; - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java index 94c2d4c0b..6f0573b02 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -362,4 +362,27 @@ public class IOUtils { org.apache.commons.io.IOUtils.closeQuietly(outputStream); } } + + /** + * Returns a file throwing a UserException if the file cannot be read. + * @param path File path + * @return LineIterator + */ + public static LineIterator lineIterator(String path) { + return lineIterator(new File(path)); + } + + /** + * Returns a file throwing a UserException if the file cannot be read. + * @param file File + * @return LineIterator + */ + public static LineIterator lineIterator(File file) { + try { + return FileUtils.lineIterator(file); + } catch (IOException e) { + throw new UserException.CouldNotReadInputFile(file, e); + } + + } } diff --git a/public/java/src/org/broadinstitute/sting/utils/yaml/FieldOrderComparator.java b/public/java/src/org/broadinstitute/sting/utils/yaml/FieldOrderComparator.java deleted file mode 100644 index 2a043466a..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/yaml/FieldOrderComparator.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.yaml; - -import org.yaml.snakeyaml.introspector.Property; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -/** - * Orders properties based on the order of the fields in the Java Bean. - */ -class FieldOrderComparator implements Comparator { - private final List propertyOrder; - - public FieldOrderComparator(Class clazz) { - propertyOrder = new ArrayList(); - for (Field field : clazz.getDeclaredFields()) - propertyOrder.add(field.getName()); - } - - @Override - public int compare(Property one, Property two) { - Integer index1 = propertyOrder.indexOf(one.getName()); - Integer index2 = propertyOrder.indexOf(two.getName()); - return index1.compareTo(index2); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/yaml/StingYamlRepresenter.java b/public/java/src/org/broadinstitute/sting/utils/yaml/StingYamlRepresenter.java deleted file mode 100644 index 157b1ce27..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/yaml/StingYamlRepresenter.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.yaml; - -import org.yaml.snakeyaml.introspector.Property; -import org.yaml.snakeyaml.nodes.*; -import org.yaml.snakeyaml.representer.Represent; -import org.yaml.snakeyaml.representer.Representer; - -import java.beans.IntrospectionException; -import java.io.File; -import java.util.Set; -import java.util.TreeSet; - -/** - * A representer with Sting prefered settings. - * - Fields are ordered in the order of the class declaration, instead of alphabetically. - * - Empty maps and sequences are not output. - * - Files are converted to their absolute paths. - */ -public class StingYamlRepresenter extends Representer { - - public StingYamlRepresenter() { - super(); - this.representers.put(File.class, new RepresentFile()); - } - - @Override - protected Set getProperties(Class type) throws IntrospectionException { - TreeSet properties = new TreeSet(new FieldOrderComparator(type)); - properties.addAll(super.getProperties(type)); - return properties; - } - - @Override - protected NodeTuple representJavaBeanProperty(Object javaBean, Property property, - Object propertyValue, Tag customTag) { - NodeTuple tuple = super.representJavaBeanProperty(javaBean, property, propertyValue, customTag); - Node valueNode = tuple.getValueNode(); - if (Tag.NULL.equals(valueNode.getTag())) { - return null;// skip 'null' values - } - if (valueNode instanceof CollectionNode) { - if (Tag.SEQ.equals(valueNode.getTag())) { - SequenceNode seq = (SequenceNode) valueNode; - if (seq.getValue().isEmpty()) { - return null;// skip empty lists - } - } - if (Tag.MAP.equals(valueNode.getTag())) { - MappingNode seq = (MappingNode) valueNode; - if (seq.getValue().isEmpty()) { - return null;// skip empty maps - } - } - } - return tuple; - } - - private class RepresentFile implements Represent { - @Override - public Node representData(Object o) { - return StingYamlRepresenter.this.representScalar(Tag.STR, ((File)o).getPath()); - } - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/yaml/YamlUtils.java b/public/java/src/org/broadinstitute/sting/utils/yaml/YamlUtils.java deleted file mode 100644 index 715c71efc..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/yaml/YamlUtils.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.yaml; - -import org.broadinstitute.sting.utils.exceptions.UserException; -import org.yaml.snakeyaml.DumperOptions; -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.constructor.Constructor; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; - -/** - * A collection of utilities for operating on YAML. - * Uses the FLOW style of writing YAML, versus the BLOCK style. - * By default uses a representer that prunes empty lists and maps. - */ -public class YamlUtils { - private static Representer representer = new StingYamlRepresenter(); - private static DumperOptions options = new DumperOptions(); - - static { - options.setCanonical(false); - options.setExplicitRoot(Tag.MAP); - options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW); - options.setPrettyFlow(true); - } - - /** - * Serialize an object to the file system. - * @param o Object to serialize. - * @param file Path to write the serialized YAML. - */ - public static void dump(Object o, File file) { - dump(o, file, representer); - } - - /** - * Serialize an object to the file system. - * @param o Object to serialize. - * @param file Path to write the serialized YAML. - * @param representer Custom representer with rules on how to serialize YAML. - */ - public static void dump(Object o, File file, Representer representer) { - Constructor constructor = new Constructor(o.getClass()); - Yaml yaml = new Yaml(constructor, representer, options); - try { - yaml.dump(o, new FileWriter(file)); - } catch (IOException ioe) { - throw new UserException.CouldNotCreateOutputFile(file, ioe); - } - } - - /** - * Deserialize an object from the file system. - * @param clazz Clazz to deserialize. - * @param file Path to read the deserialized YAML. - * @return Object deserialized from the file system. - */ - public static T load(Class clazz, File file) { - return load(clazz, file, representer); - } - - /** - * Deserialize an object from the file system. - * @param clazz Clazz to deserialize. - * @param file Path to read the deserialized YAML. - * @param representer Custom representer with rules on how to deserialize YAML. - * @return Object deserialized from the file system. - */ - @SuppressWarnings("unchecked") - public static T load(Class clazz, File file, Representer representer) { - Constructor constructor = new Constructor(clazz); - Yaml yaml = new Yaml(constructor, representer, options); - try { - return (T) yaml.load(new FileReader(file)); - } catch (IOException ioe) { - throw new UserException.CouldNotReadInputFile(file, ioe); - } - } -} diff --git a/public/java/test/org/broadinstitute/sting/BaseTest.java b/public/java/test/org/broadinstitute/sting/BaseTest.java index f99a105ae..87c03321a 100755 --- a/public/java/test/org/broadinstitute/sting/BaseTest.java +++ b/public/java/test/org/broadinstitute/sting/BaseTest.java @@ -78,7 +78,7 @@ public abstract class BaseTest { public static final String hg19Intervals = intervalsLocation + "whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.interval_list"; public static final String hg19Chr20Intervals = intervalsLocation + "whole_exome_agilent_1.1_refseq_plus_3_boosters.Homo_sapiens_assembly19.targets.chr20.interval_list"; - public static final String networkTempDir = "/broad/shptmp/"; + public static final String networkTempDir = "/broad/shptmp/" + System.getProperty("user.name") + "/"; public static final File networkTempDirFile = new File(networkTempDir); public static final File testDirFile = new File("public/testdata/"); @@ -235,6 +235,7 @@ public abstract class BaseTest { */ public static File createNetworkTempFile(String name, String extension) { try { + FileUtils.forceMkdir(networkTempDirFile); File file = File.createTempFile(name, extension, networkTempDirFile); file.deleteOnExit(); return file; diff --git a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java index f7be1d845..c9b81a9d3 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/report/GATKReportUnitTest.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.report; import org.broadinstitute.sting.BaseTest; import org.testng.Assert; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class GATKReportUnitTest extends BaseTest { @@ -45,4 +46,31 @@ public class GATKReportUnitTest extends BaseTest { Object validationReportPK = countVariants.getPrimaryKey("none.eval.none.known"); Assert.assertEquals(validationReport.get(validationReportPK, "sensitivity"), "NaN"); } + + @DataProvider(name = "rightAlignValues") + public Object[][] getRightAlignValues() { + return new Object[][] { + new Object[] {null, true}, + new Object[] {"null", true}, + new Object[] {"NA", true}, + new Object[] {"0", true}, + new Object[] {"0.0", true}, + new Object[] {"-0", true}, + new Object[] {"-0.0", true}, + new Object[] {String.valueOf(Long.MAX_VALUE), true}, + new Object[] {String.valueOf(Long.MIN_VALUE), true}, + new Object[] {String.valueOf(Float.MIN_NORMAL), true}, + new Object[] {String.valueOf(Double.MAX_VALUE), true}, + new Object[] {String.valueOf(Double.MIN_VALUE), true}, + new Object[] {String.valueOf(Double.POSITIVE_INFINITY), true}, + new Object[] {String.valueOf(Double.NEGATIVE_INFINITY), true}, + new Object[] {String.valueOf(Double.NaN), true}, + new Object[] {"hello", false} + }; + } + + @Test(dataProvider = "rightAlignValues") + public void testIsRightAlign(String value, boolean expected) { + Assert.assertEquals(GATKReportColumn.isRightAlign(value), expected, "right align of '" + value + "'"); + } } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsIntegrationTest.java index c8a25c97b..9b79653c6 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsIntegrationTest.java @@ -50,8 +50,8 @@ public class DiffObjectsIntegrationTest extends WalkerTest { @DataProvider(name = "data") public Object[][] createData() { - new TestParams(testDir + "diffTestMaster.vcf", testDir + "diffTestTest.vcf", "ed377322c615abc7dceb97025076078d"); - new TestParams(testDir + "exampleBAM.bam", testDir + "exampleBAM.simple.bam", "02e46f5d2ebb3d49570850595b3f792e"); + new TestParams(testDir + "diffTestMaster.vcf", testDir + "diffTestTest.vcf", "da3dc85a0e35a9aade5520591891b4fa"); + new TestParams(testDir + "exampleBAM.bam", testDir + "exampleBAM.simple.bam", "7dc8200730313e6753237a696296fb73"); return TestParams.getTests(TestParams.class); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java index 403ecce78..1555b56d5 100755 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/varianteval/VariantEvalIntegrationTest.java @@ -30,7 +30,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("abe943d1aac120d7e75b9b9e5dac2399") + Arrays.asList("f909fd8374f663e983b9b3fda4cf5cf1") ); executeTest("testFunctionClassWithSnpeff", spec); } @@ -50,7 +50,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("5fd9624c7a35ffb79d0feb1e233fc757") + Arrays.asList("081fcaa532c7ba8f23da739389e6f7c3") ); executeTest("testStratifySamplesAndExcludeMonomorphicSites", spec); } @@ -70,7 +70,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("4a8765cd02d36e63f6d0f0c10a6c674b") + Arrays.asList("b3852f84d07c270b8a12874083c3e31b") ); executeTest("testFundamentalsCountVariantsSNPsandIndels", spec); } @@ -91,7 +91,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("4106ab8f742ad1c3138c29220151503c") + Arrays.asList("cf70468b5ebaec408419da69b0a7fcb9") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithNovelty", spec); } @@ -113,7 +113,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("6cee3a8d68307a118944f2df5401ac89") + Arrays.asList("5e3b8b85acfc41365c8208c23abf746b") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithNoveltyAndFilter", spec); } @@ -134,7 +134,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("af5dd27354d5dfd0d2fe03149af09b55") + Arrays.asList("ccdbc50d30ece6d0d3b199c397f03ed3") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithCpG", spec); } @@ -155,7 +155,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("062a231e203671e19aa9c6507710d762") + Arrays.asList("95c690d5af8ed51573eb2f0503dcd9c2") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithFunctionalClass", spec); } @@ -176,7 +176,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("75abdd2b17c0a5e04814b6969a3d4d7e") + Arrays.asList("8e8547eb38b34bec0095b0500fd9641d") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithDegeneracy", spec); } @@ -197,7 +197,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("bdbb5f8230a4a193058750c5e506c733") + Arrays.asList("158a4651a656aea7f84c79548f6fe519") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithSample", spec); } @@ -220,7 +220,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("f076120da22930294840fcc396f5f141") + Arrays.asList("76c8a0b28d2993644120f7afa5833ab2") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithJexlExpression", spec); } @@ -245,7 +245,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("69201f4a2a7a44b38805a4aeeb8830b6") + Arrays.asList("34682193f458b93b39efac00b4fc6723") ); executeTest("testFundamentalsCountVariantsSNPsandIndelsWithMultipleJexlExpressions", spec); } @@ -264,7 +264,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("c3bd3cb6cfb21a8c2b4d5f69104bf6c2") + Arrays.asList("52f6655f1532bcea24b402010d93ce73") ); executeTest("testFundamentalsCountVariantsNoCompRod", spec); } @@ -277,7 +277,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { " --eval " + validationDataLocation + "yri.trio.gatk_glftrio.intersection.annotated.filtered.chr1.vcf" + " --comp:comp_genotypes,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.head.vcf"; WalkerTestSpec spec = new WalkerTestSpec(withSelect(tests, "DP < 50", "DP50") + " " + extraArgs + " -ST CpG -o %s", - 1, Arrays.asList("861f94e3237d62bd5bc00757319241f7")); + 1, Arrays.asList("6fa6e77f149de3d13c31d410a98043a0")); executeTestParallel("testSelect1", spec); } @@ -287,14 +287,14 @@ public class VariantEvalIntegrationTest extends WalkerTest { WalkerTestSpec spec = new WalkerTestSpec(cmdRoot + " -ST CpG --eval:VCF3 " + validationDataLocation + vcfFile + " --comp:VCF3 " + validationDataLocation + "GenotypeConcordanceComp.vcf -noEV -EV GenotypeConcordance -o %s", 1, - Arrays.asList("96f27163f16bb945f19c6623cd6db34e")); + Arrays.asList("9a56c20a7b9a554a7b530f2cb1dd776d")); executeTestParallel("testVEGenotypeConcordance" + vcfFile, spec); } @Test public void testCompVsEvalAC() { String extraArgs = "-T VariantEval -R "+b36KGReference+" -o %s -ST CpG -EV GenotypeConcordance --eval:evalYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.ug.very.few.lines.vcf --comp:compYRI,VCF3 " + validationDataLocation + "yri.trio.gatk.fake.genotypes.ac.test.vcf"; - WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("955c33365e017679047fabec0f14d5e0")); + WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("aeff16bb43be03a2a7e5b9d0108b4999")); executeTestParallel("testCompVsEvalAC",spec); } @@ -312,7 +312,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { @Test public void testCompOverlap() { String extraArgs = "-T VariantEval -R " + b37KGReference + " -L " + validationDataLocation + "VariantEval/pacbio.hg19.intervals --comp:comphapmap " + comparisonDataLocation + "Validated/HapMap/3.3/genotypes_r27_nr.b37_fwd.vcf --eval " + validationDataLocation + "VariantEval/pacbio.ts.recalibrated.vcf -noEV -EV CompOverlap -sn NA12878 -noST -ST Novelty -o %s"; - WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("fb7d989e44bd74c5376cb5732f9f3f64")); + WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("9002023b8aa8fc2c9aac58b8a79bca1e")); executeTestParallel("testCompOverlap",spec); } @@ -324,7 +324,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { " --dbsnp " + b37dbSNP132 + " --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" + " -noST -ST Novelty -o %s"; - WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("da5bcb305c5ef207ce175821efdbdefd")); + WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("38ed9d216bd43f1cceceea24146fae38")); executeTestParallel("testEvalTrackWithoutGenotypes",spec); } @@ -336,7 +336,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { " --eval:evalBI " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bi.sites.vcf" + " --eval:evalBC " + validationDataLocation + "VariantEval/ALL.20100201.chr20.bc.sites.vcf" + " -noST -ST Novelty -o %s"; - WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("fde839ece1442388f21a2f0b936756a8")); + WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("453c6b1f7165913e8b1787e22bac1281")); executeTestParallel("testMultipleEvalTracksWithoutGenotypes",spec); } @@ -353,13 +353,13 @@ public class VariantEvalIntegrationTest extends WalkerTest { " -noST -noEV -ST Novelty -EV CompOverlap" + " -o %s"; - WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("1efae6b3b88c752b771e0c8fae24464e")); + WalkerTestSpec spec = new WalkerTestSpec(extraArgs,1,Arrays.asList("61052c19211e7eb61fbbb62db5e40b56")); executeTestParallel("testMultipleCompTracks",spec); } @Test public void testPerSampleAndSubsettedSampleHaveSameResults1() { - String md5 = "bc9bcabc3105e2515d9a2d41506d2de1"; + String md5 = "0edded1cd578db62fa296c99c34a909d"; WalkerTestSpec spec = new WalkerTestSpec( buildCommandLine( @@ -414,7 +414,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("e53546243250634fc03e83b4e61ec55f") + Arrays.asList("ee22604616b3e9fc48a6dcbbf73a056d") ); executeTest("testAlleleCountStrat", spec); } @@ -435,7 +435,7 @@ public class VariantEvalIntegrationTest extends WalkerTest { "-o %s" ), 1, - Arrays.asList("c8086f0525bc13e666afeb670c2e13ae") + Arrays.asList("240369cd651c77e05e8a6659f4a6237e") ); executeTest("testIntervalStrat", spec); } diff --git a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java index 3a25bc5c1..16b6c97d0 100644 --- a/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java +++ b/public/java/test/org/broadinstitute/sting/gatk/walkers/variantutils/VCFStreamingIntegrationTest.java @@ -98,7 +98,7 @@ public class VCFStreamingIntegrationTest extends WalkerTest { " -EV CompOverlap -noEV -noST" + " -o %s", 1, - Arrays.asList("1f7ed8c0f671dd227ab764624ef0d64c") + Arrays.asList("addf5f4596ddacef40808f6d3d281111") ); executeTest("testVCFStreamingChain", selectTestSpec); diff --git a/public/java/test/org/broadinstitute/sting/pipeline/PipelineUnitTest.java b/public/java/test/org/broadinstitute/sting/pipeline/PipelineUnitTest.java deleted file mode 100644 index 891356670..000000000 --- a/public/java/test/org/broadinstitute/sting/pipeline/PipelineUnitTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2010, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.pipeline; - -import org.broadinstitute.sting.pipeline.Pipeline; -import org.broadinstitute.sting.pipeline.PipelineSample; -import org.testng.Assert; -import org.broadinstitute.sting.utils.yaml.YamlUtils; - -import org.testng.annotations.Test; - -import java.io.File; -import java.util.Map; - -public class PipelineUnitTest { - @Test - public void testDumpAndLoad() throws Exception { - Pipeline pipeline = new Pipeline(); - - pipeline.getProject().setName("PRJ_NAME"); - pipeline.getProject().setReferenceFile(new File("my.fasta")); - pipeline.getProject().setGenotypeDbsnp(new File("my.vcf")); - pipeline.getProject().setEvalDbsnp(new File("my.dbsnp")); - pipeline.getProject().getTags().put("testProjectTag", "project value here"); - - PipelineSample sample = new PipelineSample(); - sample.setId("SMP_ID"); - sample.getBamFiles().put("recalibrated", new File("recalibrated.bam")); - sample.getBamFiles().put("cleaned", new File("/absolute/path/to/cleaned.bam")); - sample.getTags().put("testSampleTag", "sample value here"); - - pipeline.getSamples().add(sample); - - File file = File.createTempFile("testDumpAndLoad", ".yaml"); - YamlUtils.dump(pipeline, file); - Pipeline pipelineLoad = YamlUtils.load(Pipeline.class, file); - - Assert.assertEquals(pipelineLoad.getProject().getName(), pipeline.getProject().getName()); - Assert.assertEquals(pipeline.getProject().getReferenceFile(), pipelineLoad.getProject().getReferenceFile()); - Assert.assertEquals(pipeline.getProject().getIntervalList(), pipelineLoad.getProject().getIntervalList()); - Assert.assertEquals(pipeline.getProject().getGenotypeDbsnp(), pipelineLoad.getProject().getGenotypeDbsnp()); - Assert.assertEquals(pipeline.getProject().getGenotypeDbsnpType(), pipelineLoad.getProject().getGenotypeDbsnpType()); - Assert.assertEquals(pipeline.getProject().getEvalDbsnp(), pipelineLoad.getProject().getEvalDbsnp()); - Assert.assertEquals(pipeline.getProject().getEvalDbsnpType(), pipelineLoad.getProject().getEvalDbsnpType()); - - Assert.assertEquals(pipelineLoad.getProject().getTags().size(), pipeline.getProject().getTags().size()); - for (Map.Entry entry : pipeline.getProject().getTags().entrySet()) - Assert.assertEquals(pipeline.getProject().getTags().get(entry.getKey()), entry.getValue()); - - Assert.assertEquals(pipelineLoad.getSamples().size(), pipeline.getSamples().size()); - for (int i = 0; i < pipeline.getSamples().size(); i++) { - PipelineSample pipelineSample = pipeline.getSamples().get(i); - PipelineSample pipelineLoadSample = pipelineLoad.getSamples().get(i); - - Assert.assertEquals(pipelineLoadSample.getId(), pipelineSample.getId()); - - Assert.assertEquals(pipelineLoadSample.getBamFiles().size(), pipelineSample.getBamFiles().size()); - for (Map.Entry entry : pipelineSample.getBamFiles().entrySet()) - Assert.assertEquals(entry.getValue(), pipelineSample.getBamFiles().get(entry.getKey())); - - Assert.assertEquals(pipelineLoadSample.getTags().size(), pipelineSample.getTags().size()); - for (Map.Entry entry : pipelineSample.getTags().entrySet()) - Assert.assertEquals(pipelineSample.getTags().get(entry.getKey()), entry.getValue()); - } - } -}