diff --git a/build.xml b/build.xml index b49c8fb4c..dcaafe322 100644 --- a/build.xml +++ b/build.xml @@ -468,7 +468,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - additionalparam="-build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet"> + additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java index 89e20dad1..5f8f19892 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java @@ -42,6 +42,7 @@ import java.util.*; * Date: 7/4/11 * Time: 12:51 PM * A generic engine for comparing tree-structured objects + * */ public class DiffEngine { final protected static Logger logger = Logger.getLogger(DiffEngine.class); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index fba6549fb..5cd99697c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -37,37 +37,86 @@ import java.io.PrintStream; import java.util.List; /** + * A generic engine for comparing tree-structured objects + * * Compares two record-oriented files, itemizing specific difference between equivalent * records in the two files. Reports both itemized and summarized differences. + * * @author Mark DePristo + * @since 7/4/11 * @version 0.1 */ @Requires(value={}) public class DiffObjectsWalker extends RodWalker { + /** + * Writes out a file of the DiffEngine format: + * + * http://www.broadinstitute.org/gsa/wiki/index.php/DiffEngine + */ @Output(doc="File to which results should be written",required=true) protected PrintStream out; - @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) - int MAX_OBJECTS_TO_READ = -1; - - @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) - int MAX_DIFFS = 0; - - @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) - int MAX_COUNT1_DIFFS = 0; - - @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) - int minCountForDiff = 1; - - @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) - boolean showItemizedDifferences = false; - + /** + * The master file against which we will compare test. This is one of the two required + * files to do the comparison. Conceptually master is the original file contained the expected + * results, but this doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="master", shortName="m", doc="Master file: expected results", required=true) File masterFile; + /** + * The test file against which we will compare to the master. This is one of the two required + * files to do the comparison. Conceptually test is the derived file from master, but this + * doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="test", shortName="t", doc="Test file: new results to compare to the master file", required=true) File testFile; + /** + * The engine will read at most this number of objects from each of master and test files. This reduces + * the memory requirements for DiffObjects but does limit you to comparing at most this number of objects + */ + @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) + int MAX_OBJECTS_TO_READ = -1; + + /** + * The max number of differences to display when summarizing. For example, if there are 10M differences, but + * maxDiffs is 10, then the comparison aborts after first ten summarized differences are shown. Note that + * the system shows differences sorted by frequency, so these 10 would be the most common between the two files. + * A value of 0 means show all possible differences. + */ + @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) + int MAX_DIFFS = 0; + + /** + * The maximum number of singleton (occurs exactly once between the two files) to display when writing out + * the summary. Only applies if maxDiffs hasn't been exceeded. For example, if maxDiffs is 10 and maxCount1Diffs + * is 2 and there are 20 diffs with count > 1, then only 10 are shown, all of which have count above 1. + */ + @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) + int MAX_COUNT1_DIFFS = 0; + + /** + * Only differences that occur more than minCountForDiff are displayed. For example, if minCountForDiff is 10, then + * a difference must occur at least 10 times between the two files to be shown. + */ + @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) + int minCountForDiff = 1; + + /** + * If provided, the system will write out the summarized, individual differences. May lead to enormous outputs, + * depending on how many differences are found. Note these are not sorted in any way, so if you have 10M + * common differences in the files, you will see 10M records, whereas the final summarize will just list the + * difference and its count of 10M. + */ + @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) + boolean showItemizedDifferences = false; + + @Argument(fullName="testEnum", doc="X", required=false) + TestEnum testEnum = TestEnum.ONE; + + public enum TestEnum { ONE, TWO }; + final DiffEngine diffEngine = new DiffEngine(); @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java new file mode 100644 index 000000000..0c575523e --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import org.broadinstitute.sting.utils.Utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by IntelliJ IDEA. + * User: depristo + * Date: 7/21/11 + * Time: 8:51 AM + * + * Common documentation information about an GATK capability + */ +public class GATKDoc { + final DocType type; + final String name; + List synonyms; + String summary, fulltext; + final Map tags; + + public final static String NA_STRING = "None provided"; + + public enum DocType { + WALKER ("Walker"), + WALKER_ARG ("Walker argument"), + READ_FILTER ("Read filter"), + ENGINE_FEATURE ("Engine feature"); + + private final String name; + DocType(String name) { + this.name = name; + } + + }; + + public GATKDoc(DocType type, String name) { + this(type, name, new ArrayList(), NA_STRING, NA_STRING, new HashMap()); + } + + public GATKDoc(DocType type, String name, List synonyms, String summary, String fulltext, Map tags) { + this.type = type; + this.name = name; + this.synonyms = synonyms; + this.summary = summary; + this.fulltext = fulltext; + this.tags = tags; + } + + public Map toDataModel() { + Map model = new HashMap(); + model.put("type", type.name); + model.put("name", name); + model.put("synonyms", Utils.join(",", synonyms)); + model.put("summary", summary); + model.put("fulltext", fulltext); + model.putAll(tags); + return model; + } + + public DocType getType() { + return type; + } + + public String getName() { + return name; + } + + public List getSynonyms() { + return synonyms; + } + + public void addSynonym(String synonyms) { + this.synonyms.add(synonyms); + } + + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + + public String getFulltext() { + return fulltext; + } + + public void setFulltext(String fulltext) { + this.fulltext = fulltext; + } + + public void addTag(String key, String value) { + this.tags.put(key, value); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index bf99be641..18b4266be 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -37,14 +37,18 @@ import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import scala.reflect.Print; +import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { + RootDoc root; + /** * Extracts the contents of certain types of javadoc and adds them to an XML file. * @param rootDoc The documentation root. @@ -64,6 +68,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { @Override protected void processDocs(RootDoc rootDoc, PrintStream ignore) { + // setup the global access to the root + root = rootDoc; + try { /* ------------------------------------------------------------------- */ /* You should do this ONLY ONCE in the whole application life-cycle: */ @@ -77,7 +84,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { cfg.setObjectWrapper(new DefaultObjectWrapper()); for ( ClassDoc doc : rootDoc.classes() ) { - if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { + if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { // && getClassName(doc).contains("UGCalcLikelihoods")) { System.out.printf("Walker class %s%n", doc); processWalkerDocs(cfg, doc); //return; @@ -135,30 +142,23 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Map> args = new HashMap>(); root.put("arguments", args); + args.put("all", new ArrayList()); args.put("required", new ArrayList()); args.put("optional", new ArrayList()); args.put("hidden", new ArrayList()); args.put("depreciated", new ArrayList()); try { for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) { + ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); + FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); + GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(argumentDataModel(argumentSource.createArgumentDefinitions().get(0))); + args.get(kind).add(doc.toDataModel()); + args.get("all").add(doc.toDataModel()); System.out.printf("Processing %s%n", argumentSource); - -// for(FieldDoc fieldDoc: classdoc.fields()) { -// //for ( AnnotationDesc desc : fieldDoc.annotations() ) { -// System.out.printf("AnnotationDesc %s%n", desc); -// if ( implementsInterface(desc.annotationType(), Argument.class, Output.class, Input.class) ) { -// (requiredAnnotation(desc) ? requiredArgs : optionalArgs).add(dataModelForArgument(desc)); -// System.out.printf("Processing %s%n", desc); -// } else { -// System.out.printf("Skipping %s%n", desc); -// } -// } -// } } } catch ( ClassNotFoundException e ) { throw new RuntimeException(e); @@ -168,19 +168,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - protected String withDefault(String val, String def) { - return val == null ? def : val; - } - - protected Map argumentDataModel(ArgumentDefinition argumentDefinition) { - Map root = new HashMap(); - root.put("shortName", withDefault(argumentDefinition.shortName, "None provided")); - root.put("required", argumentDefinition.required); - root.put("fullName", withDefault(argumentDefinition.fullName, "None provided")); - root.put("argumentType", argumentDefinition.argumentType); - root.put("doc", withDefault(argumentDefinition.doc, "None provided")); - return root; - } +// protected String withDefault(String val, String def) { +// return val == null ? def : val; +// } protected ParsingEngine createStandardGATKParsingEngine() { CommandLineProgram clp = new CommandLineGATK(); @@ -192,24 +182,66 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - protected Map dataModelForArgument(AnnotationDesc desc) { - Map root = new HashMap(); - root.put("shortName", "None provided"); - root.put("required", false); - root.put("fullName", "None provided"); - root.put("doc", "None provided"); - - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - root.put(keyvalue.element().name(), keyvalue.value().value()); - } - return root; + private FieldDoc getFieldDoc(ClassDoc classDoc, String name) { + return getFieldDoc(classDoc, name, true); } - protected boolean requiredAnnotation(AnnotationDesc desc) { - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - if ( keyvalue.element().name().equals("required") ) - return keyvalue.value().toString().equals("true"); + private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { + System.out.printf("Looking for %s in %s%n", name, classDoc.name()); + for ( FieldDoc fieldDoc : classDoc.fields(false) ) { + System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); + if ( fieldDoc.name().equals(name) ) + return fieldDoc; + + Field field = getFieldForFieldDoc(fieldDoc); + if ( field.isAnnotationPresent(ArgumentCollection.class) ) { + ClassDoc typeDoc = root.classNamed(fieldDoc.type().qualifiedTypeName()); + if ( typeDoc == null ) + throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); + else { + FieldDoc result = getFieldDoc(typeDoc, name, false); + if ( result != null ) + return result; + // else keep searching + } + } } - return false; + + // if we didn't find it here, wander up to the superclass to find the field + if ( classDoc.superclass() != null ) { + return getFieldDoc(classDoc.superclass(), name, false); + } + + if ( primary ) + throw new RuntimeException("No field found for expected field " + name); + else + return null; + } + + protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { + final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; + GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); + + if ( def.fullName != null && def.shortName != null) + doc.addSynonym("-" + def.shortName); + + doc.addTag("required", def.required ? "yes" : "no"); + doc.addTag("type", def.argumentType.getSimpleName()); + if ( def.doc != null ) doc.setSummary(def.doc); + + List attributes = new ArrayList(); + attributes.add(def.ioType.annotationClass.getSimpleName()); + if ( def.required ) attributes.add("required"); + if ( def.isFlag ) attributes.add("flag"); + if ( def.isHidden ) attributes.add("hidden"); + doc.addTag("attributes", Utils.join(",", attributes)); + + // todo -- need depreciated value + + doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); + + doc.setFulltext(fieldDoc.commentText()); + + return doc; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 05408c2fa..4f4c411a3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** @@ -191,7 +192,7 @@ public class ResourceBundleExtractorDoclet { protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { try { Class type = getClassForDoc(classDoc); - return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type)); + return lhsClass.isAssignableFrom(type) && (! requireConcrete || JVMUtils.isConcrete(type)); } catch(Throwable t) { // Ignore errors. @@ -203,6 +204,15 @@ public class ResourceBundleExtractorDoclet { return Class.forName(getClassName(doc)); } + protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { + try { + Class clazz = getClassForDoc(fieldDoc.containingClass()); + return JVMUtils.findField(clazz, fieldDoc.name()); + } catch ( ClassNotFoundException e ) { + throw new RuntimeException(e); + } + } + /** * Reconstitute the class name from the given class JavaDoc object. * @param doc the Javadoc model for the given class. diff --git a/settings/helpTemplates/test.html b/settings/helpTemplates/test.html index 6f6c222c4..f9bfc2b92 100644 --- a/settings/helpTemplates/test.html +++ b/settings/helpTemplates/test.html @@ -1,23 +1,37 @@ -<#macro argumentlist myargs> - - - - - - - <#list myargs as arg> +<#macro argumentlist name myargs> + <#if myargs?size != 0> +

${name}

+
Short nameFull nameDescription
- - - + + + + - <#-- - - --> - -
${arg.shortName}${arg.fullName}${arg.doc}NameSynonymsTypeSummary
${arg.required}
+ <#list myargs as arg> + + ${arg.name} + ${arg.synonyms} + ${arg.type} + ${arg.summary} + + <#-- + ${arg.required} + --> + + + +<#macro argumentDetails arg> +

${arg.name} / ${arg.synonyms}

+ Summary: ${arg.summary}
+ Attributes: ${arg.attributes}
+ Type: ${arg.type}
+ Options: ${arg.options}
+ Details: ${arg.fulltext}
+ + ${name} documentation @@ -34,10 +48,18 @@

Description

${description} + <#-- Create the argument summary -->

Arguments

-

Required

<@argumentlist myargs=arguments.required/> -

Optional

<@argumentlist myargs=arguments.optional/> -

Hidden

<@argumentlist myargs=arguments.hidden/> -

Depreciated

<@argumentlist myargs=arguments.depreciated/> + <@argumentlist name="Required" myargs=arguments.required/> + <@argumentlist name="Optional" myargs=arguments.optional/> + <@argumentlist name="Hidden" myargs=arguments.hidden/> + <@argumentlist name="Depreciated" myargs=arguments.depreciated/> + + <#-- Create the argument details --> +

Argument details

+ <#list arguments.all as arg> + <@argumentDetails arg=arg/> + +