gatkdoc now generalized to use @Annotation. Multiple subsystems now use annotation to receive docs

Index expanded to use summary() annotation field
UserExceptions, ReadFilters, GATK engine all use the system to generate docs
Doclet expanded to handle lots of new cases

build.xml

@@ -79,6 +79,17 @@
         <patternset refid="java.source.pattern" />
     </fileset>
 
+    <!-- terrible hack to get gatkdocs to see all files --> 
+    <patternset id="all.java.source.pattern">
+        <include name="${java.public.source.dir}/**/*.java" />
+        <include name="${java.private.source.dir}/**/*.java" />
+    </patternset>
+
+    <fileset id="all.java.source.files" dir="${basedir}">
+        <patternset refid="all.java.source.pattern" />
+    </fileset>
+
+
     <fileset id="external.source.files" dir="${external.dir}" erroronmissingdir="false">
         <include name="**/*.java" />
     </fileset>
@@ -457,9 +468,8 @@
         </javadoc>
     </target>
 
-    <target name="gatkdocs" unless="uptodate.extracthelp" depends="gatk.compile"
+    <target name="gatkdocs" unless="uptodate.extracthelp"
 	        description="Extract help key/value pair file from the JavaDoc tags.">
-        <property name="gatk.target" value="private"/>
         <path id="doclet.classpath">
             <path refid="external.dependencies" />
             <pathelement location="${java.classes}" />
@@ -472,7 +482,7 @@
                  additionalparam="-private -build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005">
             <sourcefiles>
                 <union>
-                    <fileset refid="java.source.files"/>
+                    <fileset refid="all.java.source.files"/>
                 </union>
             </sourcefiles>
         </javadoc>

public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java

@@ -50,6 +50,9 @@ import java.util.*;
  * gatk all the parsed out information.  Pretty much anything dealing with the underlying system should go here,
  * the gatk engine should  deal with any data related information.
  */
+@DocumentedGATKFeature(
+        groupName = "GATK Engine",
+        summary = "Features and arguments for the GATK engine itself, available to all walkers." )
 public class CommandLineGATK extends CommandLineExecutable {
     @Argument(fullName = "analysis_type", shortName = "T", doc = "Type of analysis to run")
     private String analysisName = null;

public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java

@@ -7,6 +7,9 @@ import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 /**
  * A SamRecordFilter that also depends on the header.
  */
+@DocumentedGATKFeature(
+        groupName = "Read filters",
+        summary = "GATK Engine arguments that filter or transfer incoming SAM/BAM data files" )
 public abstract class ReadFilter implements SamRecordFilter {
     /**
      * Sets the header for use by this filter.

public/java/src/org/broadinstitute/sting/gatk/walkers/ReadFilters.java

@@ -25,6 +25,7 @@
 package org.broadinstitute.sting.gatk.walkers;
 
 import net.sf.picard.filter.SamRecordFilter;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 
 import java.lang.annotation.*;
 

public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java

@@ -46,7 +46,9 @@ import java.util.List;
 @ReadFilters(MalformedReadFilter.class)
 @PartitionBy(PartitionType.NONE)
 @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT)
-@DocumentedGATKFeature( groupName = "GATK walkers" )
+@DocumentedGATKFeature(
+        groupName = "GATK walkers",
+        summary = "General tools available for running on the command line as part of the GATK package" )
 public abstract class Walker<MapType, ReduceType> {
     final protected static Logger logger = Logger.getLogger(Walker.class);
     private GenomeAnalysisEngine toolkit;

public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java

@@ -29,6 +29,7 @@ import net.sf.samtools.SAMRecord;
 import net.sf.samtools.SAMSequenceDictionary;
 import net.sf.samtools.SAMSequenceRecord;
 import org.broadinstitute.sting.utils.GenomeLoc;
+import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
 import org.broadinstitute.sting.utils.variantcontext.VariantContext;
 
 import java.io.File;
@@ -43,6 +44,9 @@ import java.util.Arrays;
  * Date: Sep 3, 2010
  * Time: 2:24:09 PM
  */
+@DocumentedGATKFeature(
+        groupName = "User exceptions",
+        summary = "Exceptions caused by incorrect user behavior, such as bad files, bad arguments, etc." )
 public class UserException extends ReviewedStingException {
     public UserException(String msg) { super(msg); }
     public UserException(String msg, Throwable e) { super(msg, e); }

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java

@@ -36,6 +36,8 @@ import java.lang.annotation.*;
 @Retention(RetentionPolicy.RUNTIME)
 @Target(ElementType.TYPE)
 public @interface DocumentedGATKFeature {
+    boolean enable() default true;
     String groupName();
+    String summary() default "";
     Class<? extends DocumentedGATKFeatureHandler> handler() default GenericDocumentationHandler.class;
 }

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java

@@ -25,24 +25,9 @@
 package org.broadinstitute.sting.utils.help;
 
 import com.sun.javadoc.ClassDoc;
-import com.sun.javadoc.FieldDoc;
 import com.sun.javadoc.RootDoc;
-import com.sun.javadoc.Tag;
-import freemarker.template.Configuration;
-import freemarker.template.DefaultObjectWrapper;
-import freemarker.template.Template;
-import freemarker.template.TemplateException;
-import org.broadinstitute.sting.commandline.*;
-import org.broadinstitute.sting.gatk.CommandLineGATK;
-import org.broadinstitute.sting.utils.Utils;
-import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
 
 import java.io.*;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 
 /**
  *
@@ -50,6 +35,7 @@ import java.util.Map;
 public abstract class DocumentedGATKFeatureHandler {
     private GATKDoclet doclet;
     private String groupName;
+    private DocumentedGATKFeature annotation;
 
     protected RootDoc getRootDoc() {
         return this.doclet.rootDoc;
@@ -59,10 +45,18 @@ public abstract class DocumentedGATKFeatureHandler {
         this.doclet = doclet;
     }
 
+    public DocumentedGATKFeature getAnnotation() {
+        return annotation;
+    }
+
+    public void setAnnotation(DocumentedGATKFeature annotation) {
+        this.annotation = annotation;
+    }
+
     public boolean shouldBeProcessed(ClassDoc doc) { return true; }
 
     public String getDestinationFilename(ClassDoc doc) {
-        return ResourceBundleExtractorDoclet.getClassName(doc).replace(".", "_") + ".html";
+        return HelpUtils.getClassName(doc).replace(".", "_") + ".html";
     }
 
     final public String getGroupName() {

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -109,16 +109,18 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
             cfg.setObjectWrapper(new DefaultObjectWrapper());
 
             List<DocumentationData> indexData = new ArrayList<DocumentationData>();
+            Set<DocumentedGATKFeature> docFeatures = new HashSet<DocumentedGATKFeature>();
             for ( ClassDoc doc : rootDoc.classes() ) {
                 System.out.printf("Considering %s%n", doc);
                 DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc);
                 if ( handler != null && handler.shouldBeProcessed(doc) ) {
                     DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc);
+                    docFeatures.add(handler.getAnnotation());
                     indexData.add(docData);
                 }
             }
 
-            processIndex(cfg, indexData);
+            processIndex(cfg, indexData, new ArrayList<DocumentedGATKFeature>(docFeatures));
         } catch ( FileNotFoundException e ) {
             throw new RuntimeException(e);
         } catch ( IOException e ) {
@@ -129,13 +131,19 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
     private DocumentedGATKFeatureHandler getHandlerForClassDoc(ClassDoc doc) {
         try {
             // todo -- what do I need the ? extends Object to pass the compiler?
-            Class<? extends Object> docClass = ResourceBundleExtractorDoclet.getClassForDoc(doc);
+            Class<? extends Object> docClass = HelpUtils.getClassForDoc(doc);
             if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) {
                 DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class);
+                if ( feature.enable() ) {
                     DocumentedGATKFeatureHandler handler = feature.handler().newInstance();
                     handler.setGroupName(feature.groupName());
                     handler.setDoclet(this);
+                    handler.setAnnotation(feature);
                     return handler;
+                } else {
+                    logger.info("Skipping disabled Documentation for " + doc);
+                    return null;
+                }
             } else {
                 return null; // not annotated so it shouldn't be documented
             }
@@ -152,21 +160,21 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
         }
     }
 
-    private void processIndex(Configuration cfg, List<DocumentationData> indexData) throws IOException {
+    private void processIndex(Configuration cfg, List<DocumentationData> indexData, List<DocumentedGATKFeature> docFeatures) throws IOException {
         /* Get or create a template */
         Template temp = cfg.getTemplate("generic.index.template.html");
 
         /* Merge data-model with template */
         Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html")));
         try {
-            temp.process(groupIndexData(indexData), out);
+            temp.process(groupIndexData(indexData, docFeatures), out);
             out.flush();
         } catch ( TemplateException e ) {
             throw new ReviewedStingException("Failed to create GATK documentation", e);
         }
     }
 
-    private Map<String, Object> groupIndexData(List<DocumentationData> indexData) {
+    private Map<String, Object> groupIndexData(List<DocumentationData> indexData, List<DocumentedGATKFeature> docFeatures) {
         //
         // root -> data -> { summary -> y, filename -> z }, etc
         //      -> groups -> group1, group2, etc.
@@ -175,19 +183,28 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet {
         Collections.sort(indexData);
 
         List<Map<String, String>> data = new ArrayList<Map<String, String>>();
-        Set<String> groups = new HashSet<String>();
-
         for ( DocumentationData indexDatum : indexData ) {
             data.add(indexDatum.toMap());
-            groups.add(indexDatum.group);
+        }
+
+        List<Map<String, String>> groups = new ArrayList<Map<String, String>>();
+        for ( DocumentedGATKFeature feature : docFeatures ) {
+            groups.add(toMap(feature));
         }
 
         root.put("data", data);
-        root.put("groups", new ArrayList<String>(groups));
+        root.put("groups", groups);
 
         return root;
     }
 
+    private static final Map<String, String> toMap(DocumentedGATKFeature annotation) {
+        Map<String, String> root = new HashMap<String, String>();
+        root.put("name", annotation.groupName());
+        root.put("summary", annotation.summary());
+        return root;
+    }
+
     private DocumentationData processDocumentationWithHandler(Configuration cfg,
                                                               DocumentedGATKFeatureHandler handler,
                                                               ClassDoc doc)

public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java

@@ -26,12 +26,7 @@ package org.broadinstitute.sting.utils.help;
 
 import com.sun.javadoc.ClassDoc;
 import com.sun.javadoc.FieldDoc;
-import com.sun.javadoc.RootDoc;
 import com.sun.javadoc.Tag;
-import freemarker.template.Configuration;
-import freemarker.template.DefaultObjectWrapper;
-import freemarker.template.Template;
-import freemarker.template.TemplateException;
 import org.broadinstitute.sting.commandline.*;
 import org.broadinstitute.sting.gatk.CommandLineGATK;
 import org.broadinstitute.sting.utils.Utils;
@@ -52,7 +47,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
     @Override
     public boolean shouldBeProcessed(ClassDoc doc) {
         try {
-            Class type = ResourceBundleExtractorDoclet.getClassForDoc(doc);
+            Class type = HelpUtils.getClassForDoc(doc);
             return JVMUtils.isConcrete(type);
         } catch ( ClassNotFoundException e ) {
             return false;
@@ -98,7 +93,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
         args.put("hidden", new ArrayList<Object>());
         args.put("depreciated", new ArrayList<Object>());
         try {
-            for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(ResourceBundleExtractorDoclet.getClassForDoc(classdoc)) ) {
+            for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(HelpUtils.getClassForDoc(classdoc)) ) {
                 ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0);
                 FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName());
                 GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones?
@@ -139,7 +134,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
             if ( fieldDoc.name().equals(name) )
                 return fieldDoc;
 
-            Field field = ResourceBundleExtractorDoclet.getFieldForFieldDoc(fieldDoc);
+            Field field = HelpUtils.getFieldForFieldDoc(fieldDoc);
             if ( field.isAnnotationPresent(ArgumentCollection.class) ) {
                 ClassDoc typeDoc = getRootDoc().classNamed(fieldDoc.type().qualifiedTypeName());
                 if ( typeDoc == null )

public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java

@@ -1,8 +1,33 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package org.broadinstitute.sting.utils.help;
 
 import com.sun.javadoc.FieldDoc;
 import com.sun.javadoc.PackageDoc;
 import com.sun.javadoc.ProgramElementDoc;
+import org.broadinstitute.sting.utils.classloader.JVMUtils;
 
 import java.lang.reflect.Field;
 

public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java

@@ -28,12 +28,8 @@ package org.broadinstitute.sting.utils.help;
 import com.sun.javadoc.*;
 import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.Utils;
-import org.broadinstitute.sting.utils.classloader.JVMUtils;
-import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
-import sun.tools.java.ClassNotFound;
 
 import java.io.*;
-import java.lang.reflect.Field;
 import java.util.*;
 
 /**
@@ -112,7 +108,7 @@ public class ResourceBundleExtractorDoclet {
             if(isRequiredJavadocMissing(currentClass) && isWalker(currentClass))
                 undocumentedWalkers.add(currentClass.name());
 
-            renderHelpText(getClassName(currentClass),currentClass);
+            renderHelpText(HelpUtils.getClassName(currentClass),currentClass);
         }
 
         for(PackageDoc currentPackage: packages)
@@ -177,50 +173,7 @@ public class ResourceBundleExtractorDoclet {
      * @return True if the class of the given name is a walker.  False otherwise.
      */
     protected static boolean isWalker(ClassDoc classDoc) {
-        return assignableToClass(classDoc, Walker.class, true);
+        return HelpUtils.assignableToClass(classDoc, Walker.class, true);
-    }
-
-    protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) {
-        for ( Class interfaceClass : interfaceClasses )
-            if ( assignableToClass(classDoc, interfaceClass, false) )
-                return true;
-        return false;
-    }
-
-    protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) {
-        try {
-            Class type = getClassForDoc(classDoc);
-            return lhsClass.isAssignableFrom(type) && (! requireConcrete || JVMUtils.isConcrete(type));
-        }
-        catch(Throwable t) {
-            // Ignore errors.
-            return false;
-        }
-    }
-
-    protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException {
-        return Class.forName(getClassName(doc));
-    }
-
-    protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) {
-        try {
-            Class clazz = getClassForDoc(fieldDoc.containingClass());
-            return JVMUtils.findField(clazz, fieldDoc.name());
-        } catch ( ClassNotFoundException e ) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    /**
-     * Reconstitute the class name from the given class JavaDoc object.
-     * @param doc the Javadoc model for the given class.
-     * @return The (string) class name of the given class.
-     */
-    protected static String getClassName(ProgramElementDoc doc) {
-        PackageDoc containingPackage = doc.containingPackage();
-        return containingPackage.name().length() > 0 ?
-                String.format("%s.%s",containingPackage.name(),doc.name()) :
-                String.format("%s",doc.name());
     }
 
     /**

settings/helpTemplates/generic.index.template.html

@@ -1,12 +1,15 @@
 <#macro emitGroup group>
   <table border="1" cellpadding="2">
-  <h2>${group}</h2>
+  <h2>${group.name}</h2>
+  <p>
+  ${group.summary}  
+  <p>
   <tr>
      <th>Name</th>
      <th>Summary</th>
   </tr>
     <#list data as datum>
-        <#if datum.group == group>
+        <#if datum.group == group.name>
             <tr>
                <td><a href="${datum.filename}">${datum.name}</a></td>
                <td>${datum.summary}</td>
@@ -21,7 +24,7 @@
   <title>GATK documentation index</title>
 </head>
 <body>
-  <h1>GATK documentation index<h1>
+  <h1>GATK documentation index</h1>
   <#list groups as group>
     <@emitGroup group=group/>
   </#list>