Final documented version of GATKDoclet and associated classes

-- Docs on everything.
-- Feature complete.  At this point only minor improvements and bugfixes are anticipated

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java

@@ -36,8 +36,12 @@ import java.lang.annotation.*;
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface DocumentedGATKFeature {
+    /** Should we actually document this feature, even through it's annotated? */
     public boolean enable() default true;
+    /** The overall group name (walkers, readfilters) this feature is associated with */
     public String groupName();
+    /** A human readable summary of the purpose of this group of features */
     public String summary() default "";
+    /** Are there links to other docs that we should include?  CommandLineGATK.class for walkers, for example? */
     public Class[] extraDocs() default {};
 }

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java

@@ -92,9 +92,7 @@ public abstract class DocumentedGATKFeatureHandler {
      *
      * toProcess.setHandlerContent(summary, rootMap);
      *
-     * @param rootDoc
      * @param toProcess
-     * @param all
      */
-    public abstract void processOne(RootDoc rootDoc, GATKDocWorkUnit toProcess, Set<GATKDocWorkUnit> all);
+    public abstract void processOne(GATKDocWorkUnit toProcess);
 }

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureObject.java

@@ -25,12 +25,15 @@
 package org.broadinstitute.sting.utils.help;
 
 /**
- * Documentation unit.  Effectively a class version of the DocumentedGATKFeature
+ * Documentation unit.  Effectively a class version of the DocumentedGATKFeature.
+ * Immutable data structure.
  *
  * @author depristo
  */
 class DocumentedGATKFeatureObject {
+    /** Which class are we documenting.  Specific to each class being documented */
     private final Class classToDoc;
+    /** Are we enabled? */
     private final boolean enable;
     private final String groupName, summary;
     private final Class[] extraDocs;

public/java/src/org/broadinstitute/sting/utils/help/GATKDocUtils.java

@@ -25,14 +25,33 @@
 package org.broadinstitute.sting.utils.help;
 
 public class GATKDocUtils {
+    /** The URL root for RELEASED GATKDOC units */
     public final static String URL_ROOT_FOR_RELEASE_GATKDOCS = "http://www.broadinstitute.org/gsa/gatkdocs/release/";
+    /** The URL root for STABLE GATKDOC units */
     public final static String URL_ROOT_FOR_STABLE_GATKDOCS = "http://iwww.broadinstitute.org/gsa/gatkdocs/stable/";
+    /** The URL root for UNSTABLE GATKDOC units */
     public final static String URL_ROOT_FOR_UNSTABLE_GATKDOCS = "http://iwww.broadinstitute.org/gsa/gatkdocs/unstable/";
 
+    /**
+     * Return the filename of the GATKDoc HTML that would be generated for Class.  This
+     * does not guarantee that the docs exist, or that docs would actually be generated
+     * for class (might not be annotated for documentation, for example).  But if
+     * this class is documented, GATKDocs will write the docs to a file named as returned
+     * by this function.
+     *
+     * @param c
+     * @return
+     */
     public static String htmlFilenameForClass(Class c) {
         return c.getName().replace(".", "_") + ".html";
     }
 
+    /**
+     * Returns a full URL http://etc/ linking to the documentation for class (assuming it
+     * exists).  Currently points to the RELEASE doc path only.
+     * @param c
+     * @return
+     */
     public static String helpLinksToGATKDocs(Class c) {
         String classPath = htmlFilenameForClass(c);
         StringBuilder b = new StringBuilder();

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -43,18 +43,65 @@ import java.io.*;
 import java.util.*;
 
 /**
+ * Javadoc Doclet that combines javadoc, GATK ParsingEngine annotations, and FreeMarker
+ * templates to produce html formatted GATKDocs for walkers
+ * and other classes.
  *
+ * This document has the following workflow:
+ *
+ * 1 -- walk the javadoc heirarchy, looking for class that have the
+ *   DocumentedGATKFeature annotation or are in the type heirarchy in the
+ *   static list of things to document, and are to be documented
+ * 2 -- construct for each a GATKDocWorkUnit, resulting in the complete
+ *   set of things to document
+ * 3 -- for each unit, actually generate an html page documenting it
+ *   as well as links to related features via their units.  Writing
+ *   of a specific class HTML is accomplished by a generate DocumentationHandler
+ * 4 -- write out an index of all units, organized by group
+ *
+ * The documented classes are restricted to only those with @DocumentedGATKFeature
+ * annotation or are in the STATIC_DOCS class.
  */
 public class GATKDoclet {
-    final protected static File SETTINGS_DIR = new File("settings/helpTemplates");
-    final protected static File DESTINATION_DIR = new File("gatkdocs");
     final protected static Logger logger = Logger.getLogger(GATKDoclet.class);
+
+    /** Where we find the help FreeMarker templates */
+    final protected static File SETTINGS_DIR = new File("settings/helpTemplates");
+
+    /** Where we write the GATKDoc html directory */
+    final protected static File DESTINATION_DIR = new File("gatkdocs");
+
+    // ----------------------------------------------------------------------
+    //
+    // Global variables that are set on the command line by javadoc
+    //
+    // ----------------------------------------------------------------------
     protected static String buildTimestamp = null, absoluteVersion = null;
     protected static boolean showHiddenFeatures = false;
+
     protected static boolean testOnly = false;
 
+    /**
+     * Any class that's in this list will be included in the documentation
+     * when the -test argument is provided.  Useful for debugging.
+     */
+    private static final List<Class<?>> testOnlyKeepers = Arrays.asList(
+            DocumentationTest.class, CommandLineGATK.class, UserException.class);
+
+    /** The javadoc root doc */
     RootDoc rootDoc;
 
+    /** The set of all things we are going to document */
+    Set<GATKDocWorkUnit> myWorkUnits;
+
+    /**
+     * A static list of DocumentedGATKFeatureObjects.  Any class that is as or extends
+     * one of the DocumentedGATKFeatureObjects.clazz of this collection will also
+     * be documented, even if it doesn't have the @DocumentedGATKFeature annotation.  Useful
+     * when you want to document things that implement an interface (annotations on java
+     * interfaces aren't inherited) or whose base class isn't under your control (tribble
+     * codecs).
+     */
     final static Collection<DocumentedGATKFeatureObject> STATIC_DOCS = new ArrayList<DocumentedGATKFeatureObject>();
     static {
         STATIC_DOCS.add(new DocumentedGATKFeatureObject(FeatureCodec.class,
@@ -70,7 +117,8 @@ public class GATKDoclet {
      * @throws java.io.IOException if output can't be written.
      */
     public static boolean start(RootDoc rootDoc) throws IOException {
-        logger.setLevel(Level.DEBUG);
+        logger.setLevel(Level.INFO);
+
         // load arguments
         for(String[] options: rootDoc.options()) {
             if(options[0].equals("-build-timestamp"))
@@ -83,8 +131,9 @@ public class GATKDoclet {
                 testOnly = true;
         }
 
-        GATKDoclet doclet = new GATKDoclet();
+        // process the docs
-        doclet.processDocs(rootDoc);
+        new GATKDoclet().processDocs(rootDoc);
+
         return true;
     }
 
@@ -104,17 +153,54 @@ public class GATKDoclet {
             return 0;
     }
 
+    /**
+     * Are we supposed to include @Hidden annotations in our documented output?
+     * @return
+     */
     public boolean showHiddenFeatures() {
         return showHiddenFeatures;
     }
 
-    public static boolean testOnly() {
+    /**
-        return testOnly;
+     *
+     * @param rootDoc
+     */
+    private void processDocs(RootDoc rootDoc) {
+        // setup the global access to the root
+        this.rootDoc = rootDoc;
+
+        try {
+            // basic setup
+            DESTINATION_DIR.mkdirs();
+            FileUtils.copyFile(new File(SETTINGS_DIR + "/style.css"), new File(DESTINATION_DIR + "/style.css"));
+
+            /* ------------------------------------------------------------------- */
+            /* You should do this ONLY ONCE in the whole application life-cycle:   */
+
+            Configuration cfg = new Configuration();
+            // Specify the data source where the template files come from.
+            cfg.setDirectoryForTemplateLoading(SETTINGS_DIR);
+            // Specify how templates will see the data-model. This is an advanced topic...
+            cfg.setObjectWrapper(new DefaultObjectWrapper());
+
+            myWorkUnits = computeWorkUnits();
+            for ( GATKDocWorkUnit workUnit : myWorkUnits ) {
+                processDocWorkUnit(cfg, workUnit);
             }
 
-    private static final List<Class<?>> testOnlyKeepers = Arrays.asList(
+            processIndex(cfg, new ArrayList<GATKDocWorkUnit>(myWorkUnits));
-            DocumentationTest.class, CommandLineGATK.class, UserException.class);
+        } catch ( FileNotFoundException e ) {
-    public Set<GATKDocWorkUnit> workUnits() {
+            throw new RuntimeException(e);
+        } catch ( IOException e ) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    /**
+     * Returns the set of all GATKDocWorkUnits that we are going to generate docs for.
+     * @return
+     */
+    private Set<GATKDocWorkUnit> computeWorkUnits() {
         TreeSet<GATKDocWorkUnit> m = new TreeSet<GATKDocWorkUnit>();
 
         for ( ClassDoc doc : rootDoc.classes() ) {
@@ -144,37 +230,13 @@ public class GATKDoclet {
         return m;
     }
 
-    protected void processDocs(RootDoc rootDoc) {
+    /**
-        // setup the global access to the root
+     * Create a handler capable of documenting the class doc according to feature.  Returns
-        this.rootDoc = rootDoc;
+     * null if no appropriate handler is found or doc shouldn't be documented at all.
-
+     * @param doc
-        try {
+     * @param feature
-            // basic setup
+     * @return
-            DESTINATION_DIR.mkdirs();
+     */
-            FileUtils.copyFile(new File(SETTINGS_DIR + "/style.css"), new File(DESTINATION_DIR + "/style.css"));
-
-            /* ------------------------------------------------------------------- */
-            /* You should do this ONLY ONCE in the whole application life-cycle:   */
-
-            Configuration cfg = new Configuration();
-            // Specify the data source where the template files come from.
-            cfg.setDirectoryForTemplateLoading(SETTINGS_DIR);
-            // Specify how templates will see the data-model. This is an advanced topic...
-            cfg.setObjectWrapper(new DefaultObjectWrapper());
-
-            Set<GATKDocWorkUnit> myWorkUnits = workUnits();
-            for ( GATKDocWorkUnit workUnit : myWorkUnits ) {
-                processDocWorkUnit(cfg, workUnit, myWorkUnits);
-            }
-
-            processIndex(cfg, new ArrayList<GATKDocWorkUnit>(myWorkUnits));
-        } catch ( FileNotFoundException e ) {
-            throw new RuntimeException(e);
-        } catch ( IOException e ) {
-            throw new RuntimeException(e);
-        }
-    }
-
     private DocumentedGATKFeatureHandler createHandler(ClassDoc doc, DocumentedGATKFeatureObject feature) {
         if ( feature != null ) {
             if ( feature.enable() ) {
@@ -189,6 +251,13 @@ public class GATKDoclet {
         return null;
     }
 
+    /**
+     * Returns the instantiated DocumentedGATKFeatureObject that describes the GATKDoc
+     * structure we will apply to Doc.
+     *
+     * @param doc
+     * @return null if this proves inappropriate or doc shouldn't be documented
+     */
     private DocumentedGATKFeatureObject getFeatureForClassDoc(ClassDoc doc) {
         Class<? extends Object> docClass = getClassForClassDoc(doc);
 
@@ -208,6 +277,11 @@ public class GATKDoclet {
         }
     }
 
+    /**
+     * Return the Java class described by the ClassDoc doc
+     * @param doc
+     * @return
+     */
     private Class<? extends Object> getClassForClassDoc(ClassDoc doc) {
         try {
             // todo -- what do I need the ? extends Object to pass the compiler?
@@ -223,10 +297,12 @@ public class GATKDoclet {
         }
     }
 
-    public static ClassDoc getClassDocForClass(RootDoc rootDoc, Class clazz) {
+    /**
-        return rootDoc.classNamed(clazz.getName());
+     * Create the html index listing all of the GATKDocs features
-    }
+     * @param cfg
-
+     * @param indexData
+     * @throws IOException
+     */
     private void processIndex(Configuration cfg, List<GATKDocWorkUnit> indexData) throws IOException {
         /* Get or create a template */
         Template temp = cfg.getTemplate("generic.index.template.html");
@@ -241,6 +317,12 @@ public class GATKDoclet {
         }
     }
 
+    /**
+     * Helpful function to create the html index.  Given all of the already run GATKDocWorkUnits,
+     * create the high-level grouping data listing individual features by group.
+     * @param indexData
+     * @return
+     */
     private Map<String, Object> groupIndexData(List<GATKDocWorkUnit> indexData) {
         //
         // root -> data -> { summary -> y, filename -> z }, etc
@@ -268,6 +350,11 @@ public class GATKDoclet {
         return root;
     }
 
+    /**
+     * Trivial helper routine that returns the map of name and summary given the annotation
+     * @param annotation
+     * @return
+     */
     private static final Map<String, String> toMap(DocumentedGATKFeatureObject annotation) {
         Map<String, String> root = new HashMap<String, String>();
         root.put("name", annotation.groupName());
@@ -275,18 +362,39 @@ public class GATKDoclet {
         return root;
     }
 
-    public final static GATKDocWorkUnit findWorkUnitForClass(Class c, Set<GATKDocWorkUnit> all) {
+    /**
-        for ( final GATKDocWorkUnit unit : all )
+     * Helper function that finding the GATKDocWorkUnit associated with class from among all of the work units
+     * @param c the class we are looking for
+     * @return the GATKDocWorkUnit whose .clazz.equals(c), or null if none could be found
+     */
+    public final GATKDocWorkUnit findWorkUnitForClass(Class c) {
+        for ( final GATKDocWorkUnit unit : this.myWorkUnits )
             if ( unit.clazz.equals(c) )
                 return unit;
         return null;
     }
 
-    private void processDocWorkUnit(Configuration cfg, GATKDocWorkUnit unit, Set<GATKDocWorkUnit> all)
+    /**
+     * Return the ClassDoc associated with clazz
+     * @param clazz
+     * @return
+     */
+    public ClassDoc getClassDocForClass(Class clazz) {
+        return rootDoc.classNamed(clazz.getName());
+    }
+
+    /**
+     * High-level function that processes a single DocWorkUnit unit using its handler
+     *
+     * @param cfg
+     * @param unit
+     * @throws IOException
+     */
+    private void processDocWorkUnit(Configuration cfg, GATKDocWorkUnit unit)
             throws IOException {
         //System.out.printf("Processing documentation for class %s%n", unit.classDoc);
 
-        unit.handler.processOne(rootDoc, unit, all);
+        unit.handler.processOne(unit);
 
         // Get or create a template
         Template temp = cfg.getTemplate(unit.handler.getTemplateName(unit.classDoc));

public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java

@@ -53,15 +53,15 @@ import java.util.*;
 public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
     private static Logger logger = Logger.getLogger(GenericDocumentationHandler.class);
 
+    /**
+     * The max. length of the longest of --fullName -shortName argument name
+     * before we prefer the shorter option.
+     */
+    private static final int MAX_DISPLAY_NAME = 30;
+
     /** The Class we are documenting */
     private GATKDocWorkUnit toProcess;
 
-    /** The set of all classes we are documenting, for cross-referencing */
-    private Set<GATKDocWorkUnit> all;
-
-    /** The JavaDoc root */
-    private RootDoc rootDoc;
-
     @Override
     public boolean includeInDocs(ClassDoc doc) {
         try {
@@ -79,10 +79,8 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
     }
 
     @Override
-    public void processOne(RootDoc rootDoc, GATKDocWorkUnit toProcessArg, Set<GATKDocWorkUnit> allArg) {
+    public void processOne(GATKDocWorkUnit toProcessArg) {
-        this.rootDoc = rootDoc;
         this.toProcess = toProcessArg;
-        this.all = allArg;
 
         //System.out.printf("%s class %s%n", toProcess.group, toProcess.classDoc);
         Map<String, Object> root = new HashMap<String, Object>();
@@ -126,7 +124,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
 
         // add in all of the explicitly related items
         for ( final Class extraDocClass : toProcess.annotation.extraDocs() ) {
-            final GATKDocWorkUnit otherUnit = GATKDoclet.findWorkUnitForClass(extraDocClass, all);
+            final GATKDocWorkUnit otherUnit = getDoclet().findWorkUnitForClass(extraDocClass);
             if ( otherUnit == null )
                 throw new ReviewedStingException("Requested extraDocs for class without any documentation: " + extraDocClass);
             extraDocsData.add(
@@ -388,6 +386,13 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
         return getFieldDoc(classDoc, name, true);
     }
 
+    /**
+     * Recursive helper routine to getFieldDoc()
+     * @param classDoc
+     * @param name
+     * @param primary
+     * @return
+     */
     private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) {
         //System.out.printf("Looking for %s in %s%n", name, classDoc.name());
         for ( FieldDoc fieldDoc : classDoc.fields(false) ) {
@@ -422,7 +427,14 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
             return null;
     }
 
-    private static final int MAX_DISPLAY_NAME = 30;
+    /**
+     * Returns a Pair of (main, synonym) names for argument with fullName s1 and
+     * shortName s2.  The main is selected to be the longest of the two, provided
+     * it doesn't exceed MAX_DISPLAY_NAME, in which case the shorter is taken.
+     * @param s1
+     * @param s2
+     * @return
+     */
     Pair<String, String> displayNames(String s1, String s2) {
         if ( s1 == null ) return new Pair<String, String>(s2, null);
         if ( s2 == null ) return new Pair<String, String>(s1, null);
@@ -436,6 +448,15 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
             return new Pair<String, String>(l, s);
     }
 
+    /**
+     * Returns a human readable string that describes the Type type of a GATK argument.
+     *
+     * This will include parameterized types, so that Set{T} shows up as Set(T) and not
+     * just Set in the docs.
+     *
+     * @param type
+     * @return
+     */
     protected String argumentTypeString(Type type) {
         if (type instanceof ParameterizedType) {
             ParameterizedType parameterizedType = (ParameterizedType)type;
@@ -454,6 +475,13 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
         }
     }
 
+    /**
+     * Helper routine that returns the Feature.class required by a RodBinding,
+     * either T for RodBinding{T} or List{RodBinding{T}}.  Returns null if
+     * the Type doesn't fit either model.
+     * @param type
+     * @return
+     */
     protected Class<? extends Feature> getFeatureTypeIfPossible(Type type) {
         if ( type instanceof ParameterizedType) {
             ParameterizedType paramType = (ParameterizedType)type;
@@ -471,6 +499,14 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
         return null;
     }
 
+    /**
+     * High-level entry point for creating a FreeMarker map describing the GATK argument
+     * source with definition def, with associated javadoc fieldDoc.
+     * @param fieldDoc
+     * @param source
+     * @param def
+     * @return a non-null Map binding argument keys with their values
+     */
     protected Map<String, Object> docForArgument(FieldDoc fieldDoc, ArgumentSource source, ArgumentDefinition def) {
         Map<String, Object> root = new HashMap<String, Object>();
         Pair<String, String> names = displayNames("-" + def.shortName, "--" + def.fullName);
@@ -503,27 +539,29 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
         root.put("summary", def.doc != null ? def.doc : "");
         root.put("fulltext", fieldDoc.commentText());
 
+        // What are our enum options?
+        if ( def.validOptions != null )
+            root.put("options", docForEnumArgument(source.field.getType()));
+
+        // general attributes
         List<String> attributes = new ArrayList<String>();
-        // this one below is just too much.
-        //attributes.add(def.ioType.annotationClass.getSimpleName());
         if ( def.required ) attributes.add("required");
-        // flag is just boolean, not interesting
-        //if ( def.isFlag ) attributes.add("flag");
-        if ( def.isHidden ) attributes.add("hidden");
         if ( source.isDeprecated() ) attributes.add("depreciated");
         if ( attributes.size() > 0 )
             root.put("attributes", Utils.join(", ", attributes));
 
-        if ( def.validOptions != null ) {
-            root.put("options", docForEnumArgument(source.field.getType()));
-        }
-
         return root;
     }
 
+    /**
+     * Helper routine that provides a FreeMarker map for an enumClass, grabbing the
+     * values of the enum and their associated javadoc documentation.
+     * @param enumClass
+     * @return
+     */
     @Requires("enumClass.isEnum()")
     private List<Map<String, Object>> docForEnumArgument(Class enumClass) {
-        ClassDoc doc = GATKDoclet.getClassDocForClass(rootDoc, enumClass);
+        ClassDoc doc = this.getDoclet().getClassDocForClass(enumClass);
         if ( doc == null ) //  || ! doc.isEnum() )
             throw new RuntimeException("Tried to get docs for enum " + enumClass + " but got instead: " + doc);
 
@@ -537,5 +575,4 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
 
         return bindings;
     }
-
 }

public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java

@@ -32,14 +32,6 @@ import org.broadinstitute.sting.utils.classloader.JVMUtils;
 import java.lang.reflect.Field;
 
 public class HelpUtils {
-
-    protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) {
-        for (Class interfaceClass : interfaceClasses)
-            if (assignableToClass(classDoc, interfaceClass, false))
-                return true;
-        return false;
-    }
-
     protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) {
         try {
             Class type = getClassForDoc(classDoc);