From e892489696ec01c2edda7da3ca9efe19a5d8b7f6 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 21 Jul 2011 15:20:34 -0400 Subject: [PATCH] V2 of the document system. Now uses GATKDoc class to organize documentation for arguments. Arguments now listed by feature (required, optional, hidden, etc) and link to detailed information about the argument in the html Lots of code moving between Class and ClassDoc objects. Should be refactored into a single static utility class. --- build.xml | 2 +- .../gatk/walkers/diffengine/DiffEngine.java | 1 + .../walkers/diffengine/DiffObjectsWalker.java | 79 ++++++++--- .../sting/utils/help/GATKDoc.java | 123 ++++++++++++++++++ .../sting/utils/help/GATKDoclet.java | 118 +++++++++++------ .../help/ResourceBundleExtractorDoclet.java | 12 +- settings/helpTemplates/test.html | 62 ++++++--- 7 files changed, 317 insertions(+), 80 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java diff --git a/build.xml b/build.xml index b49c8fb4c..dcaafe322 100644 --- a/build.xml +++ b/build.xml @@ -468,7 +468,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - additionalparam="-build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet"> + additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java index 89e20dad1..5f8f19892 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java @@ -42,6 +42,7 @@ import java.util.*; * Date: 7/4/11 * Time: 12:51 PM * A generic engine for comparing tree-structured objects + * */ public class DiffEngine { final protected static Logger logger = Logger.getLogger(DiffEngine.class); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index fba6549fb..5cd99697c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -37,37 +37,86 @@ import java.io.PrintStream; import java.util.List; /** + * A generic engine for comparing tree-structured objects + * * Compares two record-oriented files, itemizing specific difference between equivalent * records in the two files. Reports both itemized and summarized differences. + * * @author Mark DePristo + * @since 7/4/11 * @version 0.1 */ @Requires(value={}) public class DiffObjectsWalker extends RodWalker { + /** + * Writes out a file of the DiffEngine format: + * + * http://www.broadinstitute.org/gsa/wiki/index.php/DiffEngine + */ @Output(doc="File to which results should be written",required=true) protected PrintStream out; - @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) - int MAX_OBJECTS_TO_READ = -1; - - @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) - int MAX_DIFFS = 0; - - @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) - int MAX_COUNT1_DIFFS = 0; - - @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) - int minCountForDiff = 1; - - @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) - boolean showItemizedDifferences = false; - + /** + * The master file against which we will compare test. This is one of the two required + * files to do the comparison. Conceptually master is the original file contained the expected + * results, but this doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="master", shortName="m", doc="Master file: expected results", required=true) File masterFile; + /** + * The test file against which we will compare to the master. This is one of the two required + * files to do the comparison. Conceptually test is the derived file from master, but this + * doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="test", shortName="t", doc="Test file: new results to compare to the master file", required=true) File testFile; + /** + * The engine will read at most this number of objects from each of master and test files. This reduces + * the memory requirements for DiffObjects but does limit you to comparing at most this number of objects + */ + @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) + int MAX_OBJECTS_TO_READ = -1; + + /** + * The max number of differences to display when summarizing. For example, if there are 10M differences, but + * maxDiffs is 10, then the comparison aborts after first ten summarized differences are shown. Note that + * the system shows differences sorted by frequency, so these 10 would be the most common between the two files. + * A value of 0 means show all possible differences. + */ + @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) + int MAX_DIFFS = 0; + + /** + * The maximum number of singleton (occurs exactly once between the two files) to display when writing out + * the summary. Only applies if maxDiffs hasn't been exceeded. For example, if maxDiffs is 10 and maxCount1Diffs + * is 2 and there are 20 diffs with count > 1, then only 10 are shown, all of which have count above 1. + */ + @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) + int MAX_COUNT1_DIFFS = 0; + + /** + * Only differences that occur more than minCountForDiff are displayed. For example, if minCountForDiff is 10, then + * a difference must occur at least 10 times between the two files to be shown. + */ + @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) + int minCountForDiff = 1; + + /** + * If provided, the system will write out the summarized, individual differences. May lead to enormous outputs, + * depending on how many differences are found. Note these are not sorted in any way, so if you have 10M + * common differences in the files, you will see 10M records, whereas the final summarize will just list the + * difference and its count of 10M. + */ + @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) + boolean showItemizedDifferences = false; + + @Argument(fullName="testEnum", doc="X", required=false) + TestEnum testEnum = TestEnum.ONE; + + public enum TestEnum { ONE, TWO }; + final DiffEngine diffEngine = new DiffEngine(); @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java new file mode 100644 index 000000000..0c575523e --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import org.broadinstitute.sting.utils.Utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by IntelliJ IDEA. + * User: depristo + * Date: 7/21/11 + * Time: 8:51 AM + * + * Common documentation information about an GATK capability + */ +public class GATKDoc { + final DocType type; + final String name; + List synonyms; + String summary, fulltext; + final Map tags; + + public final static String NA_STRING = "None provided"; + + public enum DocType { + WALKER ("Walker"), + WALKER_ARG ("Walker argument"), + READ_FILTER ("Read filter"), + ENGINE_FEATURE ("Engine feature"); + + private final String name; + DocType(String name) { + this.name = name; + } + + }; + + public GATKDoc(DocType type, String name) { + this(type, name, new ArrayList(), NA_STRING, NA_STRING, new HashMap()); + } + + public GATKDoc(DocType type, String name, List synonyms, String summary, String fulltext, Map tags) { + this.type = type; + this.name = name; + this.synonyms = synonyms; + this.summary = summary; + this.fulltext = fulltext; + this.tags = tags; + } + + public Map toDataModel() { + Map model = new HashMap(); + model.put("type", type.name); + model.put("name", name); + model.put("synonyms", Utils.join(",", synonyms)); + model.put("summary", summary); + model.put("fulltext", fulltext); + model.putAll(tags); + return model; + } + + public DocType getType() { + return type; + } + + public String getName() { + return name; + } + + public List getSynonyms() { + return synonyms; + } + + public void addSynonym(String synonyms) { + this.synonyms.add(synonyms); + } + + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + + public String getFulltext() { + return fulltext; + } + + public void setFulltext(String fulltext) { + this.fulltext = fulltext; + } + + public void addTag(String key, String value) { + this.tags.put(key, value); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index bf99be641..18b4266be 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -37,14 +37,18 @@ import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import scala.reflect.Print; +import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { + RootDoc root; + /** * Extracts the contents of certain types of javadoc and adds them to an XML file. * @param rootDoc The documentation root. @@ -64,6 +68,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { @Override protected void processDocs(RootDoc rootDoc, PrintStream ignore) { + // setup the global access to the root + root = rootDoc; + try { /* ------------------------------------------------------------------- */ /* You should do this ONLY ONCE in the whole application life-cycle: */ @@ -77,7 +84,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { cfg.setObjectWrapper(new DefaultObjectWrapper()); for ( ClassDoc doc : rootDoc.classes() ) { - if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { + if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { // && getClassName(doc).contains("UGCalcLikelihoods")) { System.out.printf("Walker class %s%n", doc); processWalkerDocs(cfg, doc); //return; @@ -135,30 +142,23 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Map> args = new HashMap>(); root.put("arguments", args); + args.put("all", new ArrayList()); args.put("required", new ArrayList()); args.put("optional", new ArrayList()); args.put("hidden", new ArrayList()); args.put("depreciated", new ArrayList()); try { for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) { + ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); + FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); + GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(argumentDataModel(argumentSource.createArgumentDefinitions().get(0))); + args.get(kind).add(doc.toDataModel()); + args.get("all").add(doc.toDataModel()); System.out.printf("Processing %s%n", argumentSource); - -// for(FieldDoc fieldDoc: classdoc.fields()) { -// //for ( AnnotationDesc desc : fieldDoc.annotations() ) { -// System.out.printf("AnnotationDesc %s%n", desc); -// if ( implementsInterface(desc.annotationType(), Argument.class, Output.class, Input.class) ) { -// (requiredAnnotation(desc) ? requiredArgs : optionalArgs).add(dataModelForArgument(desc)); -// System.out.printf("Processing %s%n", desc); -// } else { -// System.out.printf("Skipping %s%n", desc); -// } -// } -// } } } catch ( ClassNotFoundException e ) { throw new RuntimeException(e); @@ -168,19 +168,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - protected String withDefault(String val, String def) { - return val == null ? def : val; - } - - protected Map argumentDataModel(ArgumentDefinition argumentDefinition) { - Map root = new HashMap(); - root.put("shortName", withDefault(argumentDefinition.shortName, "None provided")); - root.put("required", argumentDefinition.required); - root.put("fullName", withDefault(argumentDefinition.fullName, "None provided")); - root.put("argumentType", argumentDefinition.argumentType); - root.put("doc", withDefault(argumentDefinition.doc, "None provided")); - return root; - } +// protected String withDefault(String val, String def) { +// return val == null ? def : val; +// } protected ParsingEngine createStandardGATKParsingEngine() { CommandLineProgram clp = new CommandLineGATK(); @@ -192,24 +182,66 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - protected Map dataModelForArgument(AnnotationDesc desc) { - Map root = new HashMap(); - root.put("shortName", "None provided"); - root.put("required", false); - root.put("fullName", "None provided"); - root.put("doc", "None provided"); - - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - root.put(keyvalue.element().name(), keyvalue.value().value()); - } - return root; + private FieldDoc getFieldDoc(ClassDoc classDoc, String name) { + return getFieldDoc(classDoc, name, true); } - protected boolean requiredAnnotation(AnnotationDesc desc) { - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - if ( keyvalue.element().name().equals("required") ) - return keyvalue.value().toString().equals("true"); + private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { + System.out.printf("Looking for %s in %s%n", name, classDoc.name()); + for ( FieldDoc fieldDoc : classDoc.fields(false) ) { + System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); + if ( fieldDoc.name().equals(name) ) + return fieldDoc; + + Field field = getFieldForFieldDoc(fieldDoc); + if ( field.isAnnotationPresent(ArgumentCollection.class) ) { + ClassDoc typeDoc = root.classNamed(fieldDoc.type().qualifiedTypeName()); + if ( typeDoc == null ) + throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); + else { + FieldDoc result = getFieldDoc(typeDoc, name, false); + if ( result != null ) + return result; + // else keep searching + } + } } - return false; + + // if we didn't find it here, wander up to the superclass to find the field + if ( classDoc.superclass() != null ) { + return getFieldDoc(classDoc.superclass(), name, false); + } + + if ( primary ) + throw new RuntimeException("No field found for expected field " + name); + else + return null; + } + + protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { + final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; + GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); + + if ( def.fullName != null && def.shortName != null) + doc.addSynonym("-" + def.shortName); + + doc.addTag("required", def.required ? "yes" : "no"); + doc.addTag("type", def.argumentType.getSimpleName()); + if ( def.doc != null ) doc.setSummary(def.doc); + + List attributes = new ArrayList(); + attributes.add(def.ioType.annotationClass.getSimpleName()); + if ( def.required ) attributes.add("required"); + if ( def.isFlag ) attributes.add("flag"); + if ( def.isHidden ) attributes.add("hidden"); + doc.addTag("attributes", Utils.join(",", attributes)); + + // todo -- need depreciated value + + doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); + + doc.setFulltext(fieldDoc.commentText()); + + return doc; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 05408c2fa..4f4c411a3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** @@ -191,7 +192,7 @@ public class ResourceBundleExtractorDoclet { protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { try { Class type = getClassForDoc(classDoc); - return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type)); + return lhsClass.isAssignableFrom(type) && (! requireConcrete || JVMUtils.isConcrete(type)); } catch(Throwable t) { // Ignore errors. @@ -203,6 +204,15 @@ public class ResourceBundleExtractorDoclet { return Class.forName(getClassName(doc)); } + protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { + try { + Class clazz = getClassForDoc(fieldDoc.containingClass()); + return JVMUtils.findField(clazz, fieldDoc.name()); + } catch ( ClassNotFoundException e ) { + throw new RuntimeException(e); + } + } + /** * Reconstitute the class name from the given class JavaDoc object. * @param doc the Javadoc model for the given class. diff --git a/settings/helpTemplates/test.html b/settings/helpTemplates/test.html index 6f6c222c4..f9bfc2b92 100644 --- a/settings/helpTemplates/test.html +++ b/settings/helpTemplates/test.html @@ -1,23 +1,37 @@ -<#macro argumentlist myargs> - - - - - - - <#list myargs as arg> +<#macro argumentlist name myargs> + <#if myargs?size != 0> +

${name}

+
Short nameFull nameDescription
- - - + + + + - <#-- - - --> - -
${arg.shortName}${arg.fullName}${arg.doc}NameSynonymsTypeSummary
${arg.required}
+ <#list myargs as arg> + + ${arg.name} + ${arg.synonyms} + ${arg.type} + ${arg.summary} + + <#-- + ${arg.required} + --> + + + +<#macro argumentDetails arg> +

${arg.name} / ${arg.synonyms}

+ Summary: ${arg.summary}
+ Attributes: ${arg.attributes}
+ Type: ${arg.type}
+ Options: ${arg.options}
+ Details: ${arg.fulltext}
+ + ${name} documentation @@ -34,10 +48,18 @@

Description

${description} + <#-- Create the argument summary -->

Arguments

-

Required

<@argumentlist myargs=arguments.required/> -

Optional

<@argumentlist myargs=arguments.optional/> -

Hidden

<@argumentlist myargs=arguments.hidden/> -

Depreciated

<@argumentlist myargs=arguments.depreciated/> + <@argumentlist name="Required" myargs=arguments.required/> + <@argumentlist name="Optional" myargs=arguments.optional/> + <@argumentlist name="Hidden" myargs=arguments.hidden/> + <@argumentlist name="Depreciated" myargs=arguments.depreciated/> + + <#-- Create the argument details --> +

Argument details

+ <#list arguments.all as arg> + <@argumentDetails arg=arg/> + +