Generalized the documentation system to use a class-specific annotation and processor.

Need to generalize and bug fix the system.  But at a high level it's working now.

public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java

@@ -34,6 +34,7 @@ import org.broadinstitute.sting.gatk.walkers.Attribution;
 import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.help.ApplicationDetails;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 import org.broadinstitute.sting.utils.text.TextFormattingUtils;
 
 import java.util.*;

public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java

@@ -2,6 +2,7 @@ package org.broadinstitute.sting.gatk.filters;
 
 import net.sf.picard.filter.SamRecordFilter;
 import org.broadinstitute.sting.gatk.GenomeAnalysisEngine;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 
 /**
  * A SamRecordFilter that also depends on the header.

public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java

@@ -31,6 +31,8 @@ import org.broadinstitute.sting.gatk.filters.MalformedReadFilter;
 import org.broadinstitute.sting.utils.GenomeLoc;
 import org.broadinstitute.sting.utils.baq.BAQ;
 import org.broadinstitute.sting.utils.collections.Pair;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
+import org.broadinstitute.sting.utils.help.WalkerDocumentationHandler;
 
 import java.util.List;
 
@@ -44,6 +46,7 @@ import java.util.List;
 @ReadFilters(MalformedReadFilter.class)
 @PartitionBy(PartitionType.NONE)
 @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT)
+@DocumentedGATKFeature( handler = WalkerDocumentationHandler.class )
 public abstract class Walker<MapType, ReduceType> {
     final protected static Logger logger = Logger.getLogger(Walker.class);
     private GenomeAnalysisEngine toolkit;

public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/CpG.java

@@ -8,6 +8,18 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 import java.util.ArrayList;
 import java.util.Set;
 
+/**
+ * CpG is a stratification module for VariantEval that divides the input data by within/not within a CpG site
+ *
+ * <p>
+ * It is a three-state stratification:
+ * <ul>
+ *     <li>The locus is a CpG site ("CpG")
+ *     <li>The locus is not a CpG site ("non_CpG")
+ *     <li>The locus is either a CpG or not a CpG site ("all")
+ * </ul>
+ * A CpG site is defined as a site where the reference base at a locus is a C and the adjacent reference base in the 3' direction is a G.
+ */
 public class CpG extends VariantStratifier {
     private ArrayList<String> states;
 

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.help;
+
+import java.lang.annotation.*;
+
+/**
+ * An annotation to identify a class as a GATK capability for documentation
+ *
+ * @author depristo
+ */
+@Documented
+@Inherited
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.TYPE)
+public @interface DocumentedGATKFeature {
+    Class<? extends DocumentedGATKFeatureHandler> handler();
+}

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.help;
+
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.FieldDoc;
+import com.sun.javadoc.RootDoc;
+import com.sun.javadoc.Tag;
+import freemarker.template.Configuration;
+import freemarker.template.DefaultObjectWrapper;
+import freemarker.template.Template;
+import freemarker.template.TemplateException;
+import org.broadinstitute.sting.commandline.*;
+import org.broadinstitute.sting.gatk.CommandLineGATK;
+import org.broadinstitute.sting.utils.Utils;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public abstract class DocumentedGATKFeatureHandler {
+    private GATKDoclet doclet;
+
+    protected RootDoc getRootDoc() {
+        return this.doclet.rootDoc;
+    }
+
+    public void setDoclet(GATKDoclet doclet) {
+        this.doclet = doclet;
+    }
+
+    public boolean shouldBeProcessed(ClassDoc doc) { return true; }
+
+    public String getDestinationFilename(ClassDoc doc) {
+        return ResourceBundleExtractorDoclet.getClassName(doc).replace(".", "_") + ".html";
+    }
+
+    public abstract String getGroupName();
+    public abstract String getTemplateName(ClassDoc doc) throws IOException;
+    public abstract GATKDoclet.DocumentationData processOne(ClassDoc doc);
+}

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -24,29 +24,57 @@
 
 package org.broadinstitute.sting.utils.help;
 
-import com.sun.javadoc.*;
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.RootDoc;
 import freemarker.template.Configuration;
 import freemarker.template.DefaultObjectWrapper;
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
-import org.broadinstitute.sting.commandline.*;
+import org.apache.log4j.Logger;
-import org.broadinstitute.sting.gatk.CommandLineExecutable;
-import org.broadinstitute.sting.gatk.CommandLineGATK;
-import org.broadinstitute.sting.gatk.walkers.Walker;
-import org.broadinstitute.sting.utils.Utils;
-import org.broadinstitute.sting.utils.classloader.JVMUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
-import scala.reflect.Print;
-import sun.tools.java.ClassNotFound;
 
 import java.io.*;
-import java.lang.reflect.Field;
 import java.util.*;
 
 /**
  *
  */
 public class GATKDoclet extends ResourceBundleExtractorDoclet {
+    final protected static Logger logger = Logger.getLogger(GATKDoclet.class);
+
+    public static class DocumentationData implements Comparable<DocumentationData> {
+        String name, summary, filename;
+        Map<String, Object> forTemplate;
+        String group;
+
+        public DocumentationData(String name, String summary, Map<String, Object> forTemplate) {
+            this.name = name;
+            this.summary = summary;
+            this.forTemplate = forTemplate;
+        }
+
+        public void setGroup(String group) {
+            this.group = group;
+        }
+
+        public void setFilename(String filename) {
+            this.filename = filename;
+        }
+
+        public Map<String, String> toMap() {
+            Map<String, String> data = new HashMap<String, String>();
+            data.put("name", name);
+            data.put("summary", summary);
+            data.put("filename", filename);
+            data.put("group", group);
+            return data;
+        }
+
+        public int compareTo(DocumentationData other) {
+            return this.name.compareTo(other.name);
+        }
+    }
+
     RootDoc rootDoc;
 
     /**
@@ -57,7 +85,6 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
      */
     public static boolean start(RootDoc rootDoc) throws IOException {
         GATKDoclet doclet = new GATKDoclet();
-        //PrintStream out = doclet.loadData(rootDoc, false);
         doclet.processDocs(rootDoc, null);
         return true;
     }
@@ -77,20 +104,20 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
 
             Configuration cfg = new Configuration();
             // Specify the data source where the template files come from.
-            // Here I set a file directory for it:
             cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/"));
             // Specify how templates will see the data-model. This is an advanced topic...
-            // but just use this:
             cfg.setObjectWrapper(new DefaultObjectWrapper());
 
-            List<Map<String, Object>> indexData = new ArrayList<Map<String, Object>>();
+            List<DocumentationData> indexData = new ArrayList<DocumentationData>();
             for ( ClassDoc doc : rootDoc.classes() ) {
-                if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { //  && getClassName(doc).contains("UGCalcLikelihoods")) {
+                DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc);
-                    System.out.printf("Walker class %s%n", doc);
+                if ( handler != null && handler.shouldBeProcessed(doc) ) {
-                    indexData.add(processWalkerDocs(cfg, doc));
+                    DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc);
+                    indexData.add(docData);
                 }
             }
-            processWalkerIndex(indexData,cfg);
+
+            processIndex(cfg, indexData);
         } catch ( FileNotFoundException e ) {
             throw new RuntimeException(e);
         } catch ( IOException e ) {
@@ -98,171 +125,91 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
         }
     }
 
-    private void processWalkerIndex(List<Map<String, Object>> indexData, Configuration cfg) throws IOException {
+    private DocumentedGATKFeatureHandler getHandlerForClassDoc(ClassDoc doc) {
+        try {
+            // todo -- what do I need the ? extends Object to pass the compiler?
+            Class<? extends Object> docClass = ResourceBundleExtractorDoclet.getClassForDoc(doc);
+            if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) {
+                DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class);
+                DocumentedGATKFeatureHandler handler = feature.handler().newInstance();
+                handler.setDoclet(this);
+                return handler;
+            } else {
+                return null; // not annotated so it shouldn't be documented
+            }
+        } catch ( ClassNotFoundException e) {
+            //logger.warn("Couldn't find class for ClassDoc " + doc);
+            // we got a classdoc for a class we can't find.  Maybe in a library or something
+            return null;
+        } catch ( IllegalAccessException e) {
+            throw new RuntimeException(e); // the constructor is now private -- this is an error
+        } catch ( InstantiationException e) {
+            throw new RuntimeException(e); // the constructor is now private -- this is an error
+        } catch ( UnsatisfiedLinkError e) {
+            return null; // naughty BWA bindings
+        }
+    }
+
+    private void processIndex(Configuration cfg, List<DocumentationData> indexData) throws IOException {
         /* Get or create a template */
         Template temp = cfg.getTemplate("walker.index.template.html");
 
         /* Merge data-model with template */
         Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html")));
         try {
-            Map<String, Object> root = new HashMap<String, Object>();
+            temp.process(groupIndexData(indexData), out);
-            root.put("walkers", indexData);
-            temp.process(root, out);
             out.flush();
         } catch ( TemplateException e ) {
             throw new ReviewedStingException("Failed to create GATK documentation", e);
         }
     }
 
-    private Map<String, Object> processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException {
+    private Map<String, Object> groupIndexData(List<DocumentationData> indexData) {
-        // Create the root hash
+        //
-        Map root = buildWalkerDataModel(doc);
+        // root -> data -> { summary -> y, filename -> z }, etc
-
+        //      -> groups -> group1, group2, etc.
-        /* Get or create a template */
-        Template temp = cfg.getTemplate("walker.template.html");
-
-        /* Merge data-model with template */
-        File outputFile = new File(getClassName(doc).replace(".", "_") + ".html");
-        File outputPath = new File("testdoc/" + outputFile);
-        try {
-            Writer out = new OutputStreamWriter(new FileOutputStream(outputPath));
-            temp.process(root, out);
-            out.flush();
-        } catch ( TemplateException e ) {
-            throw new ReviewedStingException("Failed to create GATK documentation", e);
-        }
-
-        // add index data
-        Map<String, Object> indexData = new HashMap<String, Object>();
-        indexData.put("filename", outputFile.toString());
-        indexData.put("name", doc.name());
-        indexData.put("summary", root.get("summary"));
-        return indexData;
-    }
-
-
-    private Map buildWalkerDataModel(ClassDoc classdoc) {
         Map<String, Object> root = new HashMap<String, Object>();
 
-        root.put("name", classdoc.name());
+        Collections.sort(indexData);
 
-        // Extract overrides from the doc tags.
+        List<Map<String, String>> data = new ArrayList<Map<String, String>>();
-        StringBuilder summaryBuilder = new StringBuilder();
+        Set<String> groups = new HashSet<String>();
-        for(Tag tag: classdoc.firstSentenceTags())
-             summaryBuilder.append(tag.text());
-        root.put("summary", summaryBuilder.toString());
-        root.put("description", classdoc.commentText());
 
-        for(Tag tag: classdoc.tags()) {
+        for ( DocumentationData indexDatum : indexData ) {
-            root.put(tag.name(), tag.text());
+            data.add(indexDatum.toMap());
+            groups.add(indexDatum.group);
         }
 
-        ParsingEngine parsingEngine = createStandardGATKParsingEngine();
+        root.put("data", data);
-//        for (ArgumentDefinition argumentDefinition : parsingEngine.argumentDefinitions )
+        root.put("groups", new ArrayList<String>(groups));
-//            System.out.println(argumentDefinition);
 
-        Map<String, List<Object>> args = new HashMap<String, List<Object>>();
-        root.put("arguments", args);
-        args.put("all", new ArrayList<Object>());
-        args.put("required", new ArrayList<Object>());
-        args.put("optional", new ArrayList<Object>());
-        args.put("hidden", new ArrayList<Object>());
-        args.put("depreciated", new ArrayList<Object>());
-        try {
-            for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) {
-                ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0);
-                FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName());
-                GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones?
-                String kind = "optional";
-                if ( argumentSource.isRequired() ) kind = "required";
-                else if ( argumentSource.isHidden() ) kind = "hidden";
-                else if ( argumentSource.isDeprecated() ) kind = "depreciated";
-                args.get(kind).add(doc.toDataModel());
-                args.get("all").add(doc.toDataModel());
-                System.out.printf("Processing %s%n", argumentSource);
-            }
-        } catch ( ClassNotFoundException e ) {
-            throw new RuntimeException(e);
-        }
-
-        System.out.printf("Root is %s%n", root);
         return root;
     }
 
-//    protected String withDefault(String val, String def) {
+    private DocumentationData processDocumentationWithHandler(Configuration cfg,
-//        return val == null ? def : val;
+                                                              DocumentedGATKFeatureHandler handler,
-//    }
+                                                              ClassDoc doc)
+            throws IOException {
+        System.out.printf("Processing documentation for class %s%n", doc);
 
-    protected ParsingEngine createStandardGATKParsingEngine() {
+        DocumentationData docData = handler.processOne(doc);
-        CommandLineProgram clp = new CommandLineGATK();
+        docData.setGroup(handler.getGroupName());
+
+        // Get or create a template
+        Template temp = cfg.getTemplate(handler.getTemplateName(doc));
+
+        // Merge data-model with template
+        String filename = handler.getDestinationFilename(doc);
+        docData.setFilename(filename);
+        File outputPath = new File("testdoc/" + filename);
         try {
-            CommandLineProgram.start(clp, new String[]{}, true);
+            Writer out = new OutputStreamWriter(new FileOutputStream(outputPath));
-            return clp.parser;
+            temp.process(docData.forTemplate, out);
-        } catch (Exception e) {
+            out.flush();
-            throw new RuntimeException(e);
+        } catch ( TemplateException e ) {
-        }
+            throw new ReviewedStingException("Failed to create GATK documentation", e);
-    }
-
-    private FieldDoc getFieldDoc(ClassDoc classDoc, String name) {
-        return getFieldDoc(classDoc, name, true);
-    }
-
-    private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) {
-        System.out.printf("Looking for %s in %s%n", name, classDoc.name());
-        for ( FieldDoc fieldDoc : classDoc.fields(false) ) {
-            System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name());
-            if ( fieldDoc.name().equals(name) )
-                return fieldDoc;
-
-            Field field = getFieldForFieldDoc(fieldDoc);
-            if ( field.isAnnotationPresent(ArgumentCollection.class) ) {
-                ClassDoc typeDoc = this.rootDoc.classNamed(fieldDoc.type().qualifiedTypeName());
-                if ( typeDoc == null )
-                    throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc");
-                else {
-                    FieldDoc result = getFieldDoc(typeDoc, name, false);
-                    if ( result != null )
-                        return result;
-                    // else keep searching
-                }
-            }
         }
 
-        // if we didn't find it here, wander up to the superclass to find the field
+        return docData;
-        if ( classDoc.superclass() != null ) {
-            return getFieldDoc(classDoc.superclass(), name, false);
-        }
-
-        if ( primary )
-            throw new RuntimeException("No field found for expected field " + name);
-        else
-            return null;
-    }
-
-    protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) {
-        final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName;
-        GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name);
-
-        if ( def.fullName != null && def.shortName != null)
-            doc.addSynonym("-" + def.shortName);
-
-        doc.addTag("required", def.required ? "yes" : "no");
-        doc.addTag("type", def.argumentType.getSimpleName());
-        if ( def.doc != null ) doc.setSummary(def.doc);
-
-        List<String> attributes = new ArrayList<String>();
-        attributes.add(def.ioType.annotationClass.getSimpleName());
-        if ( def.required ) attributes.add("required");
-        if ( def.isFlag ) attributes.add("flag");
-        if ( def.isHidden ) attributes.add("hidden");
-        doc.addTag("attributes", Utils.join(",", attributes));
-
-        // todo -- need depreciated value
-
-        doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions));
-
-        doc.setFulltext(fieldDoc.commentText());
-
-        return doc;
     }
 }

public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java

@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.utils.help;
+
+import com.sun.javadoc.ClassDoc;
+import com.sun.javadoc.FieldDoc;
+import com.sun.javadoc.RootDoc;
+import com.sun.javadoc.Tag;
+import freemarker.template.Configuration;
+import freemarker.template.DefaultObjectWrapper;
+import freemarker.template.Template;
+import freemarker.template.TemplateException;
+import org.broadinstitute.sting.commandline.*;
+import org.broadinstitute.sting.gatk.CommandLineGATK;
+import org.broadinstitute.sting.utils.Utils;
+import org.broadinstitute.sting.utils.classloader.JVMUtils;
+import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+
+import java.io.*;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public class WalkerDocumentationHandler extends DocumentedGATKFeatureHandler {
+    @Override
+    public boolean shouldBeProcessed(ClassDoc doc) {
+        try {
+            Class type = ResourceBundleExtractorDoclet.getClassForDoc(doc);
+            return JVMUtils.isConcrete(type);
+        } catch ( ClassNotFoundException e ) {
+            return false;
+        }
+    }
+
+    @Override
+    public String getGroupName() { return "GATK Walkers"; }
+
+    @Override
+    public String getTemplateName(ClassDoc doc) throws IOException {
+        return "walker.template.html";
+    }
+
+    @Override
+    public GATKDoclet.DocumentationData processOne(ClassDoc doc) {
+        System.out.printf("Walker class %s%n", doc);
+        Map<String, Object> root = buildWalkerDataModel(doc); // Create the root hash
+        return new GATKDoclet.DocumentationData(doc.name(), (String)root.get("summary"), root);
+    }
+
+
+    private Map<String, Object> buildWalkerDataModel(ClassDoc classdoc) {
+        Map<String, Object> root = new HashMap<String, Object>();
+
+        root.put("name", classdoc.name());
+
+        // Extract overrides from the doc tags.
+        StringBuilder summaryBuilder = new StringBuilder();
+        for(Tag tag: classdoc.firstSentenceTags())
+             summaryBuilder.append(tag.text());
+        root.put("summary", summaryBuilder.toString());
+        root.put("description", classdoc.commentText());
+
+        for(Tag tag: classdoc.tags()) {
+            root.put(tag.name(), tag.text());
+        }
+
+        ParsingEngine parsingEngine = createStandardGATKParsingEngine();
+
+        Map<String, List<Object>> args = new HashMap<String, List<Object>>();
+        root.put("arguments", args);
+        args.put("all", new ArrayList<Object>());
+        args.put("required", new ArrayList<Object>());
+        args.put("optional", new ArrayList<Object>());
+        args.put("hidden", new ArrayList<Object>());
+        args.put("depreciated", new ArrayList<Object>());
+        try {
+            for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(ResourceBundleExtractorDoclet.getClassForDoc(classdoc)) ) {
+                ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0);
+                FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName());
+                GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones?
+                String kind = "optional";
+                if ( argumentSource.isRequired() ) kind = "required";
+                else if ( argumentSource.isHidden() ) kind = "hidden";
+                else if ( argumentSource.isDeprecated() ) kind = "depreciated";
+                args.get(kind).add(doc.toDataModel());
+                args.get("all").add(doc.toDataModel());
+                System.out.printf("Processing %s%n", argumentSource);
+            }
+        } catch ( ClassNotFoundException e ) {
+            throw new RuntimeException(e);
+        }
+
+        //System.out.printf("Root is %s%n", root);
+        return root;
+    }
+
+    protected ParsingEngine createStandardGATKParsingEngine() {
+        CommandLineProgram clp = new CommandLineGATK();
+        try {
+            CommandLineProgram.start(clp, new String[]{}, true);
+            return clp.parser;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private FieldDoc getFieldDoc(ClassDoc classDoc, String name) {
+        return getFieldDoc(classDoc, name, true);
+    }
+
+    private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) {
+        System.out.printf("Looking for %s in %s%n", name, classDoc.name());
+        for ( FieldDoc fieldDoc : classDoc.fields(false) ) {
+            System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name());
+            if ( fieldDoc.name().equals(name) )
+                return fieldDoc;
+
+            Field field = ResourceBundleExtractorDoclet.getFieldForFieldDoc(fieldDoc);
+            if ( field.isAnnotationPresent(ArgumentCollection.class) ) {
+                ClassDoc typeDoc = getRootDoc().classNamed(fieldDoc.type().qualifiedTypeName());
+                if ( typeDoc == null )
+                    throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc");
+                else {
+                    FieldDoc result = getFieldDoc(typeDoc, name, false);
+                    if ( result != null )
+                        return result;
+                    // else keep searching
+                }
+            }
+        }
+
+        // if we didn't find it here, wander up to the superclass to find the field
+        if ( classDoc.superclass() != null ) {
+            return getFieldDoc(classDoc.superclass(), name, false);
+        }
+
+        if ( primary )
+            throw new RuntimeException("No field found for expected field " + name);
+        else
+            return null;
+    }
+
+    protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) {
+        final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName;
+        GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name);
+
+        if ( def.fullName != null && def.shortName != null)
+            doc.addSynonym("-" + def.shortName);
+
+        doc.addTag("required", def.required ? "yes" : "no");
+        doc.addTag("type", def.argumentType.getSimpleName());
+        if ( def.doc != null ) doc.setSummary(def.doc);
+
+        List<String> attributes = new ArrayList<String>();
+        attributes.add(def.ioType.annotationClass.getSimpleName());
+        if ( def.required ) attributes.add("required");
+        if ( def.isFlag ) attributes.add("flag");
+        if ( def.isHidden ) attributes.add("hidden");
+        doc.addTag("attributes", Utils.join(",", attributes));
+
+        // todo -- need depreciated value
+
+        doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions));
+
+        doc.setFulltext(fieldDoc.commentText());
+
+        return doc;
+    }
+}

settings/helpTemplates/walker.index.template.html

@@ -1,21 +1,30 @@
-<html>
+<#macro emitGroup group>
-<head>
-  <title>GATK walker index</title>
-</head>
-<body>
-  <h1>GATK walker index<h1>
   <table border="1" cellpadding="2">
+  <h2>${group}</h2>
   <tr>
      <th>Name</th>
      <th>Summary</th>
   </tr>
-  <#list walkers as walker>
+    <#list data as datum>
-    <tr>
+        <#if datum.group == group>
-       <td><a href="${walker.filename}">${walker.name}</a></td>
+            <tr>
-       <td>${walker.summary}</td>
+               <td><a href="${datum.filename}">${datum.name}</a></td>
-    </tr>
+               <td>${datum.summary}</td>
-  </#list>
+            </tr>
+        </#if>
+    </#list>
   </table>
+</#macro>   
+
+<html>
+<head>
+  <title>GATK documentation index</title>
+</head>
+<body>
+  <h1>GATK documentation index<h1>
+  <#list groups as group>
+    <@emitGroup group=group/>
+  </#list>
 </body>
 </html>