Completely hacked together version of a FreeMarker + javadoc + custom doclet walker documentation generator

build.xml

@@ -457,6 +457,26 @@
         </javadoc>
     </target>
 
+    <target name="walkerdocs" unless="uptodate.extracthelp"
+	        description="Extract help key/value pair file from the JavaDoc tags.">
+        <path id="doclet.classpath">
+            <path refid="external.dependencies" />
+            <pathelement location="${java.classes}" />
+        </path>
+
+        <javadoc doclet="org.broadinstitute.sting.utils.help.GATKDoclet"
+                 docletpathref="doclet.classpath"
+                 classpathref="external.dependencies"
+                 classpath="${java.classes}"
+                 additionalparam="-build-timestamp &quot;${build.timestamp}&quot; -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet">
+            <sourcefiles>
+                <union>
+                    <fileset refid="java.source.files"/>
+                </union>
+            </sourcefiles>
+        </javadoc>
+    </target>
+
     <target name="sting.compile" depends="gatk.compile, scala.compile" />
 
     <target name="init.jar" depends="sting.compile,extracthelp">

public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java

@@ -43,7 +43,7 @@ import java.util.Locale;
 public abstract class CommandLineProgram {
 
     /** The command-line program and the arguments it returned. */
-    protected ParsingEngine parser = null;
+    public ParsingEngine parser = null;
 
     /** the default log level */
     @Argument(fullName = "logging_level",
@@ -144,6 +144,11 @@ public abstract class CommandLineProgram {
 
     public static int result = -1;
 
+    @SuppressWarnings("unchecked")
+    public static void start(CommandLineProgram clp, String[] args) throws Exception {
+        start(clp, args, false);
+    }
+
     /**
      * This function is called to start processing the command line, and kick
      * off the execute message of the program.
@@ -153,7 +158,7 @@ public abstract class CommandLineProgram {
      * @throws Exception when an exception occurs
      */
     @SuppressWarnings("unchecked")
-    public static void start(CommandLineProgram clp, String[] args) throws Exception {
+    public static void start(CommandLineProgram clp, String[] args, boolean dryRun) throws Exception {
 
         try {
             // setup our log layout
@@ -180,8 +185,9 @@ public abstract class CommandLineProgram {
                 //   - InvalidArgument in case these arguments are specified by plugins.
                 //   - MissingRequiredArgument in case the user requested help.  Handle that later, once we've
                 //                             determined the full complement of arguments.
-                parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument,
+                if ( ! dryRun )
-                                           ParsingEngine.ValidationType.InvalidArgument));
+                    parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument,
+                            ParsingEngine.ValidationType.InvalidArgument));
                 parser.loadArgumentsIntoObject(clp);
 
                 // Initialize the logger using the loaded command line.
@@ -195,36 +201,40 @@ public abstract class CommandLineProgram {
                 if (isHelpPresent(parser))
                     printHelpAndExit(clp, parser);
 
-                parser.validate();
+                if ( ! dryRun ) parser.validate();
             } else {
                 parser.parse(args);
 
-                if (isHelpPresent(parser))
+                if ( ! dryRun ) {
-                    printHelpAndExit(clp, parser);
+                    if (isHelpPresent(parser))
+                        printHelpAndExit(clp, parser);
 
-                parser.validate();
+                    parser.validate();
+                }
                 parser.loadArgumentsIntoObject(clp);
 
                 // Initialize the logger using the loaded command line.
                 clp.setupLoggerLevel(layout);
             }
 
-            // if they specify a log location, output our data there
+            if ( ! dryRun ) {
-            if (clp.toFile != null) {
+                // if they specify a log location, output our data there
-                FileAppender appender;
+                if (clp.toFile != null) {
-                try {
+                    FileAppender appender;
-                    appender = new FileAppender(layout, clp.toFile, false);
+                    try {
-                    logger.addAppender(appender);
+                        appender = new FileAppender(layout, clp.toFile, false);
-                } catch (IOException e) {
+                        logger.addAppender(appender);
-                    throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists");
+                    } catch (IOException e) {
+                        throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists");
+                    }
                 }
+
+                // regardless of what happens next, generate the header information
+                HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
+
+                // call the execute
+                CommandLineProgram.result = clp.execute();
             }
-
-            // regardless of what happens next, generate the header information
-            HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args);
-
-            // call the execute
-            CommandLineProgram.result = clp.execute();
         }
         catch (ArgumentException e) {
             clp.parser.printHelp(clp.getApplicationDetails());

public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java

@@ -45,7 +45,7 @@ public class ParsingEngine {
      * A list of defined arguments against which command lines are matched.
      * Package protected for testing access.
      */
-    ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions();
+    public ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions();
 
     /**
      * A list of matches from defined arguments to command-line text.

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -29,10 +29,14 @@ import freemarker.template.Configuration;
 import freemarker.template.DefaultObjectWrapper;
 import freemarker.template.Template;
 import freemarker.template.TemplateException;
+import org.broadinstitute.sting.commandline.*;
+import org.broadinstitute.sting.gatk.CommandLineExecutable;
+import org.broadinstitute.sting.gatk.CommandLineGATK;
 import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.classloader.JVMUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import scala.reflect.Print;
 
 import java.io.*;
 import java.util.*;
@@ -40,7 +44,7 @@ import java.util.*;
 /**
  *
  */
-public class GATKDoclet {
+public class GATKDoclet extends ResourceBundleExtractorDoclet {
     /**
      * Extracts the contents of certain types of javadoc and adds them to an XML file.
      * @param rootDoc The documentation root.
@@ -48,47 +52,164 @@ public class GATKDoclet {
      * @throws java.io.IOException if output can't be written.
      */
     public static boolean start(RootDoc rootDoc) throws IOException {
-        /* ------------------------------------------------------------------- */
+        GATKDoclet doclet = new GATKDoclet();
-        /* You should do this ONLY ONCE in the whole application life-cycle:   */
+        //PrintStream out = doclet.loadData(rootDoc, false);
+        doclet.processDocs(rootDoc, null);
+        return true;
+    }
 
-        Configuration cfg = new Configuration();
+    public static int optionLength(String option) {
-        // Specify the data source where the template files come from.
+        return ResourceBundleExtractorDoclet.optionLength(option);
-        // Here I set a file directory for it:
+    }
-        cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/"));
-        // Specify how templates will see the data-model. This is an advanced topic...
-        // but just use this:
-        cfg.setObjectWrapper(new DefaultObjectWrapper());
 
+    @Override
+    protected void processDocs(RootDoc rootDoc, PrintStream ignore) {
+        try {
+            /* ------------------------------------------------------------------- */
+            /* You should do this ONLY ONCE in the whole application life-cycle:   */
 
+            Configuration cfg = new Configuration();
+            // Specify the data source where the template files come from.
+            // Here I set a file directory for it:
+            cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/"));
+            // Specify how templates will see the data-model. This is an advanced topic...
+            // but just use this:
+            cfg.setObjectWrapper(new DefaultObjectWrapper());
+
+            for ( ClassDoc doc : rootDoc.classes() ) {
+                if ( ResourceBundleExtractorDoclet.isWalker(doc) ) {
+                    System.out.printf("Walker class %s%n", doc);
+                    processWalkerDocs(cfg, doc);
+                    //return;
+                }
+//                else
+//                    System.out.printf("Excluding non-walker class %s%n", doc);
+            }
+        } catch ( FileNotFoundException e ) {
+            throw new RuntimeException(e);
+        } catch ( IOException e ) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private void processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException {
         /* ------------------------------------------------------------------- */
         /* You usually do these for many times in the application life-cycle:  */
 
-        /* Create a data-model */
         // Create the root hash
-        Map root = new HashMap();
+        Map root = buildWalkerDataModel(doc);
-        // Put string ``user'' into the root
-        root.put("user", "Mark DePristo");
 
         /* Get or create a template */
         Template temp = cfg.getTemplate("test.html");
 
         /* Merge data-model with template */
-        Writer out = new OutputStreamWriter(System.out);
+        Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/" + getClassName(doc).replace(".", "_") + ".html")));
         try {
             temp.process(root, out);
             out.flush();
         } catch ( TemplateException e ) {
             throw new ReviewedStingException("Failed to create GATK documentation", e);
         }
-        return true;
     }
 
-    /**
+
-     * Validate the given options against options supported by this doclet.
+    private Map buildWalkerDataModel(ClassDoc classdoc) {
-     * @param option Option to validate.
+        Map<String, Object> root = new HashMap<String, Object>();
-     * @return Number of potential parameters; 0 if not supported.
+
-     */
+        root.put("name", classdoc.name());
-    public static int optionLength(String option) {
+
-        return 0;
+        // Extract overrides from the doc tags.
+        StringBuilder summaryBuilder = new StringBuilder();
+        for(Tag tag: classdoc.firstSentenceTags())
+             summaryBuilder.append(tag.text());
+        root.put("summary", summaryBuilder.toString());
+        root.put("description", classdoc.commentText());
+
+        for(Tag tag: classdoc.tags()) {
+            root.put(tag.name(), tag.text());
+        }
+
+        ParsingEngine parsingEngine = createStandardGATKParsingEngine();
+//        for (ArgumentDefinition argumentDefinition : parsingEngine.argumentDefinitions )
+//            System.out.println(argumentDefinition);
+
+        Map<String, List<Object>> args = new HashMap<String, List<Object>>();
+        root.put("arguments", args);
+        args.put("required", new ArrayList<Object>());
+        args.put("optional", new ArrayList<Object>());
+        args.put("hidden", new ArrayList<Object>());
+        args.put("depreciated", new ArrayList<Object>());
+        try {
+            for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) {
+                String kind = "optional";
+                if ( argumentSource.isRequired() ) kind = "required";
+                else if ( argumentSource.isHidden() ) kind = "hidden";
+                else if ( argumentSource.isDeprecated() ) kind = "depreciated";
+                args.get(kind).add(argumentDataModel(argumentSource.createArgumentDefinitions().get(0)));
+                System.out.printf("Processing %s%n", argumentSource);
+
+//            for(FieldDoc fieldDoc: classdoc.fields()) {
+//                //for ( AnnotationDesc desc : fieldDoc.annotations() ) {
+//                    System.out.printf("AnnotationDesc %s%n", desc);
+//                    if ( implementsInterface(desc.annotationType(), Argument.class, Output.class, Input.class) ) {
+//                        (requiredAnnotation(desc) ? requiredArgs : optionalArgs).add(dataModelForArgument(desc));
+//                        System.out.printf("Processing %s%n", desc);
+//                    } else {
+//                        System.out.printf("Skipping %s%n", desc);
+//                    }
+//                }
+//            }
+            }
+        } catch ( ClassNotFoundException e ) {
+            throw new RuntimeException(e);
+        }
+
+        System.out.printf("Root is %s%n", root);
+        return root;
+    }
+
+    protected String withDefault(String val, String def) {
+        return val == null ? def : val;
+    }
+
+    protected Map<String, Object> argumentDataModel(ArgumentDefinition argumentDefinition) {
+        Map<String, Object> root = new HashMap<String, Object>();
+        root.put("shortName", withDefault(argumentDefinition.shortName, "None provided"));
+        root.put("required", argumentDefinition.required);
+        root.put("fullName", withDefault(argumentDefinition.fullName, "None provided"));
+        root.put("argumentType", argumentDefinition.argumentType);
+        root.put("doc", withDefault(argumentDefinition.doc, "None provided"));
+        return root;
+    }
+
+    protected ParsingEngine createStandardGATKParsingEngine() {
+        CommandLineProgram clp = new CommandLineGATK();
+        try {
+            CommandLineProgram.start(clp, new String[]{}, true);
+            return clp.parser;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected Map<String, Object> dataModelForArgument(AnnotationDesc desc) {
+        Map<String, Object> root = new HashMap<String, Object>();
+        root.put("shortName", "None provided");
+        root.put("required", false);
+        root.put("fullName", "None provided");
+        root.put("doc", "None provided");
+
+        for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) {
+            root.put(keyvalue.element().name(), keyvalue.value().value());
+        }
+        return root;
+    }
+
+    protected boolean requiredAnnotation(AnnotationDesc desc) {
+        for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) {
+            if ( keyvalue.element().name().equals("required") )
+                return keyvalue.value().toString().equals("true");
+        }
+        return false;
     }
 }

public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java

@@ -30,12 +30,10 @@ import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.Utils;
 import org.broadinstitute.sting.utils.classloader.JVMUtils;
 import org.broadinstitute.sting.utils.exceptions.ReviewedStingException;
+import sun.tools.java.ClassNotFound;
 
 import java.io.*;
-import java.util.HashSet;
+import java.util.*;
-import java.util.Properties;
-import java.util.Scanner;
-import java.util.Set;
 
 /**
  * Extracts certain types of javadoc (specifically package and class descriptions) and makes them available
@@ -48,17 +46,19 @@ public class ResourceBundleExtractorDoclet {
     /**
      * Taglet for the particular version number.
      */
-    private static final String VERSION_TAGLET_NAME = "version";
+    protected static final String VERSION_TAGLET_NAME = "version";
 
     /**
      * Maintains a collection of resources in memory as they're accumulated.
      */
-    private static final Properties resourceText = new Properties();
+    protected final Properties resourceText = new Properties();
 
     /**
      * Maintains a collection of classes that should really be documented.
      */
-    private static final Set<String> undocumentedWalkers = new HashSet<String>();
+    protected final Set<String> undocumentedWalkers = new HashSet<String>();
+
+    protected String buildTimestamp = null, versionPrefix = null, versionSuffix = null, absoluteVersion = null;
 
     /**
      * Extracts the contents of certain types of javadoc and adds them to an XML file.
@@ -67,13 +67,26 @@ public class ResourceBundleExtractorDoclet {
      * @throws IOException if output can't be written.
      */
     public static boolean start(RootDoc rootDoc) throws IOException {
+        ResourceBundleExtractorDoclet doclet = new ResourceBundleExtractorDoclet();
+        PrintStream out = doclet.loadData(rootDoc, true);
+        doclet.processDocs(rootDoc, out);
+        return true;
+    }
+
+    protected PrintStream loadData(RootDoc rootDoc, boolean overwriteResourcesFile) {
         PrintStream out = System.out;
-        String buildTimestamp = null, versionPrefix = null, versionSuffix = null, absoluteVersion = null;
 
         for(String[] options: rootDoc.options()) {
             if(options[0].equals("-out")) {
-                loadExistingResourceFile(options[1], rootDoc);
+                try {
-                out = new PrintStream(options[1]);
+                    loadExistingResourceFile(options[1], rootDoc);
+                    if ( overwriteResourcesFile )
+                        out = new PrintStream(options[1]);
+                } catch ( FileNotFoundException e ) {
+                    throw new RuntimeException(e);
+                } catch ( IOException e ) {
+                    throw new RuntimeException(e);
+                }
             }
             if(options[0].equals("-build-timestamp"))
                 buildTimestamp = options[1];
@@ -86,7 +99,10 @@ public class ResourceBundleExtractorDoclet {
         }
 
         resourceText.setProperty("build.timestamp",buildTimestamp);
+        return out;
+    }
 
+    protected void processDocs(RootDoc rootDoc, PrintStream out) {
         // Cache packages as we see them, since there's no direct way to iterate over packages.
         Set<PackageDoc> packages = new HashSet<PackageDoc>();
 
@@ -97,13 +113,19 @@ public class ResourceBundleExtractorDoclet {
             if(isRequiredJavadocMissing(currentClass) && isWalker(currentClass))
                 undocumentedWalkers.add(currentClass.name());
 
-            renderHelpText(getClassName(currentClass),currentClass,versionPrefix,versionSuffix,absoluteVersion);
+            renderHelpText(getClassName(currentClass),currentClass);
         }
 
         for(PackageDoc currentPackage: packages)
-            renderHelpText(currentPackage.name(),currentPackage,versionPrefix,versionSuffix,absoluteVersion);
+            renderHelpText(currentPackage.name(),currentPackage);
 
-        resourceText.store(out,"Strings displayed by the Sting help system");
+        try {
+            resourceText.store(out,"Strings displayed by the Sting help system");
+        } catch ( FileNotFoundException e ) {
+            throw new RuntimeException(e);
+        } catch ( IOException e ) {
+            throw new RuntimeException(e);
+        }
 
         // ASCII codes for making text blink
         final String blink = "\u001B\u005B\u0035\u006D";
@@ -111,8 +133,6 @@ public class ResourceBundleExtractorDoclet {
 
         if(undocumentedWalkers.size() > 0)
             Utils.warnUser(String.format("The following walkers are currently undocumented: %s%s%s", blink, Utils.join(" ",undocumentedWalkers), reset));
-
-        return true;
     }
 
     /**
@@ -137,7 +157,7 @@ public class ResourceBundleExtractorDoclet {
      * @throws IOException       if there is an I/O-related error other than FileNotFoundException
      *                           while attempting to read the resource file.
      */
-    private static void loadExistingResourceFile( String resourceFileName, RootDoc rootDoc ) throws IOException {
+    private void loadExistingResourceFile( String resourceFileName, RootDoc rootDoc ) throws IOException {
         try {
             BufferedReader resourceFile = new BufferedReader(new FileReader(resourceFileName));
             try {
@@ -157,10 +177,21 @@ public class ResourceBundleExtractorDoclet {
      * @param classDoc the type of the given class.
      * @return True if the class of the given name is a walker.  False otherwise.
      */
-    private static boolean isWalker(ClassDoc classDoc) {
+    protected static boolean isWalker(ClassDoc classDoc) {
+        return assignableToClass(classDoc, Walker.class, true);
+    }
+
+    protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) {
+        for ( Class interfaceClass : interfaceClasses )
+            if ( assignableToClass(classDoc, interfaceClass, false) )
+                return true;
+        return false;
+    }
+
+    protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) {
         try {
-            Class type = Class.forName(getClassName(classDoc));
+            Class type = getClassForDoc(classDoc);
-            return Walker.class.isAssignableFrom(type) && JVMUtils.isConcrete(type);
+            return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type));
         }
         catch(Throwable t) {
             // Ignore errors.
@@ -168,16 +199,20 @@ public class ResourceBundleExtractorDoclet {
         }
     }
 
+    protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException {
+        return Class.forName(getClassName(doc));
+    }
+
     /**
      * Reconstitute the class name from the given class JavaDoc object.
-     * @param classDoc the Javadoc model for the given class.
+     * @param doc the Javadoc model for the given class.
      * @return The (string) class name of the given class.
      */
-    private static String getClassName(ClassDoc classDoc) {
+    protected static String getClassName(ProgramElementDoc doc) {
-        PackageDoc containingPackage = classDoc.containingPackage();
+        PackageDoc containingPackage = doc.containingPackage();
         return containingPackage.name().length() > 0 ?
-                String.format("%s.%s",containingPackage.name(),classDoc.name()) :
+                String.format("%s.%s",containingPackage.name(),doc.name()) :
-                String.format("%s",classDoc.name());
+                String.format("%s",doc.name());
     }
 
     /**
@@ -193,10 +228,8 @@ public class ResourceBundleExtractorDoclet {
      * Renders all the help text required for a given name.
      * @param elementName element name to use as the key
      * @param element Doc element to process.
-     * @param versionPrefix Text to add to the start of the version string.
-     * @param versionSuffix Text to add to the end of the version string.
      */
-    private static void renderHelpText(String elementName, Doc element, String versionPrefix, String versionSuffix, String absoluteVersion) {
+    private void renderHelpText(String elementName, Doc element) {
         // Extract overrides from the doc tags.
         String name = null;
         String version = null;

settings/helpTemplates/test.html

@@ -0,0 +1,43 @@
+<#macro argumentlist myargs>
+    <table border="1" span=100>
+    <tr>
+        <th>Short name</th>
+        <th>Full name</th>
+        <th>Description</th>
+    </tr>
+    <#list myargs as arg>
+        <tr>
+            <td>${arg.shortName}</td>
+            <td>${arg.fullName}</td>
+            <td>${arg.doc}</td>
+        </tr>
+        <#--
+            <td>${arg.required}</td>
+        -->
+    </#list>
+    </table>
+</#macro>    
+   
+<html>
+<head>
+  <title>${name} documentation</title>
+</head>
+<body>
+  <h1>${name}<h1>
+  <h2>Summary</h2>
+  ${summary}
+  <h2>Version</h2>
+  ${version!"unknown version"}
+  <#if author??>
+      <h2>Author</h2>
+      ${author}
+  </#if>
+  <h2>Description</h2>
+  ${description}
+  <h2>Arguments</h2>
+  <h3>Required</h3> <@argumentlist myargs=arguments.required/>
+  <h3>Optional</h3> <@argumentlist myargs=arguments.optional/>
+  <h3>Hidden</h3> <@argumentlist myargs=arguments.hidden/>
+  <h3>Depreciated</h3> <@argumentlist myargs=arguments.depreciated/>
+</body>
+</html>