Merge branch 'master' into rodRefactor

public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java

@@ -36,6 +36,7 @@ import org.broadinstitute.sting.gatk.walkers.Walker;
 import org.broadinstitute.sting.utils.exceptions.UserException;
 import org.broadinstitute.sting.utils.help.ApplicationDetails;
 import org.broadinstitute.sting.utils.help.DocumentedGATKFeature;
+import org.broadinstitute.sting.utils.help.GATKDoclet;
 import org.broadinstitute.sting.utils.text.TextFormattingUtils;
 
 import java.util.*;
@@ -175,12 +176,8 @@ public class CommandLineGATK extends CommandLineExecutable {
         StringBuilder additionalHelp = new StringBuilder();
         Formatter formatter = new Formatter(additionalHelp);
 
-        formatter.format("Description:%n");
+        formatter.format("For a full description of this walker, see its GATKdocs at:%n");
-
+        formatter.format("%s%n", GATKDoclet.helpLinksToGATKDocs(walkerType));
-        WalkerManager walkerManager = engine.getWalkerManager();
-        String walkerHelpText = walkerManager.getWalkerDescriptionText(walkerType);
-
-        printDescriptorLine(formatter,WALKER_INDENT,"",WALKER_INDENT,FIELD_SEPARATOR,walkerHelpText,TextFormattingUtils.DEFAULT_LINE_WIDTH);
 
         return additionalHelp.toString();
     }
@@ -194,8 +191,6 @@ public class CommandLineGATK extends CommandLineExecutable {
         StringBuilder additionalHelp = new StringBuilder();
         Formatter formatter = new Formatter(additionalHelp);
 
-        formatter.format("Available analyses:%n");
-
         // Get the list of walker names from the walker manager.
         WalkerManager walkerManager = engine.getWalkerManager();
 

public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibratorArgumentCollection.java

@@ -50,7 +50,7 @@ public class VariantRecalibratorArgumentCollection {
     @Argument(fullName="numKMeans", shortName="nKM", doc="The number of k-means iterations to perform in order to initialize the means of the Gaussians in the Gaussian mixture model.", required=false)
     public int NUM_KMEANS_ITERATIONS = 30;
     @Argument(fullName="stdThreshold", shortName="std", doc="If a variant has annotations more than -std standard deviations away from mean then don't use it for building the Gaussian mixture model.", required=false)
-    public double STD_THRESHOLD = 8.0;
+    public double STD_THRESHOLD = 14.0;
     @Argument(fullName="qualThreshold", shortName="qual", doc="If a known variant has raw QUAL value less than -qual then don't use it for building the Gaussian mixture model.", required=false)
     public double QUAL_THRESHOLD = 80.0;
     @Argument(fullName="shrinkage", shortName="shrinkage", doc="The shrinkage parameter in variational Bayes algorithm.", required=false)

public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java

@@ -31,29 +31,70 @@ import java.io.*;
 import java.util.Set;
 
 /**
- *
+ * Extend this class to provide a documentation handler for GATKdocs
  */
 public abstract class DocumentedGATKFeatureHandler {
     private GATKDoclet doclet;
 
+    /**
+     * @return the javadoc RootDoc of this javadoc run
+     */
     protected RootDoc getRootDoc() {
         return this.doclet.rootDoc;
     }
 
+    /** Set the master doclet driving this handler */
     public void setDoclet(GATKDoclet doclet) {
         this.doclet = doclet;
     }
 
+    /**
+     * @return the GATKDoclet driving this documentation run
+     */
     public GATKDoclet getDoclet() {
         return doclet;
     }
 
-    public boolean shouldBeProcessed(ClassDoc doc) { return true; }
+    /**
+     * Should return false iff this handler wants GATKDoclet to skip documenting
+     * this ClassDoc.
+     * @param doc that is being considered for inclusion in the docs
+     * @return true if the doclet should document ClassDoc doc
+     */
+    public boolean includeInDocs(ClassDoc doc) { return true; }
 
-    public String getDestinationFilename(ClassDoc doc) {
+    /**
-        return HelpUtils.getClassName(doc).replace(".", "_") + ".html";
+     * Return the flat filename (no paths) that the handler would like the Doclet to
+     * write out the documentation for ClassDoc doc and its associated Class clazz
+     * @param doc
+     * @param clazz
+     * @return
+     */
+    public String getDestinationFilename(ClassDoc doc, Class clazz) {
+        return GATKDoclet.htmlFilenameForClass(clazz);
     }
 
+    /**
+     * Return the name of the FreeMarker template we will use to process ClassDoc doc.
+     *
+     * Note this is a flat filename relative to settings/helpTemplates in the GATK source tree
+     * @param doc
+     * @return
+     * @throws IOException
+     */
     public abstract String getTemplateName(ClassDoc doc) throws IOException;
+
+    /**
+     * Actually generate the documentation map associated with toProcess
+     *
+     * Can use all to provide references and rootDoc for additional information, if necessary.
+     * Implementing methods should end with a call to setHandlerContext on toProcess, as in:
+     *
+     * toProcess.setHandlerContent(summary, rootMap);
+     *
+     * @param rootDoc
+     * @param toProcess
+     * @param all
+     */
     public abstract void processOne(RootDoc rootDoc, GATKDocWorkUnit toProcess, Set<GATKDocWorkUnit> all);
 }

public/java/src/org/broadinstitute/sting/utils/help/GATKDocWorkUnit.java

@@ -30,19 +30,29 @@ import java.util.HashMap;
 import java.util.Map;
 
 /**
-* Created by IntelliJ IDEA.
+ * Simple collection of all relevant information about something the GATKDoclet can document
-* User: depristo
+ *
-* Date: 7/24/11
+ * Created by IntelliJ IDEA.
-* Time: 7:59 PM
+ * User: depristo
-* To change this template use File | Settings | File Templates.
+ * Date: 7/24/11
-*/
+ * Time: 7:59 PM
-public class GATKDocWorkUnit implements Comparable<GATKDocWorkUnit> {
+ */
-    // known at the start
+class GATKDocWorkUnit implements Comparable<GATKDocWorkUnit> {
-    final String name, filename, group;
+    /** The class that's being documented */
-    final DocumentedGATKFeatureHandler handler;
-    final ClassDoc classDoc;
     final Class clazz;
+    /** The name of the thing we are documenting */
+    final String name;
+    /** the filename where we will be writing the docs for this class */
+    final String filename;
+    /** The name of the documentation group (e.g., walkers, read filters) class belongs to */
+    final String group;
+    /** The documentation handler for this class */
+    final DocumentedGATKFeatureHandler handler;
+    /** The javadoc documentation for clazz */
+    final ClassDoc classDoc;
+    /** The annotation that lead to this Class being in GATKDoc */
     final DocumentedGATKFeature annotation;
+    /** When was this walker built, and what's the absolute version number */
     final String buildTimestamp, absoluteVersion;
 
     // set by the handler
@@ -64,12 +74,21 @@ public class GATKDocWorkUnit implements Comparable<GATKDocWorkUnit> {
         this.absoluteVersion = absoluteVersion;
     }
 
+    /**
+     * Called by the GATKDoclet to set handler provided context for this work unit
+     * @param summary
+     * @param forTemplate
+     */
     public void setHandlerContent(String summary, Map<String, Object> forTemplate) {
         this.summary = summary;
         this.forTemplate = forTemplate;
     }
 
-    public Map<String, String> toMap() {
+    /**
+     * Return a String -> String map suitable for FreeMarker to create an index to this WorkUnit
+     * @return
+     */
+    public Map<String, String> indexDataMap() {
         Map<String, String> data = new HashMap<String, String>();
         data.put("name", name);
         data.put("summary", summary);
@@ -78,6 +97,11 @@ public class GATKDocWorkUnit implements Comparable<GATKDocWorkUnit> {
         return data;
     }
 
+    /**
+     * Sort in order of the name of this WorkUnit
+     * @param other
+     * @return
+     */
     public int compareTo(GATKDocWorkUnit other) {
         return this.name.compareTo(other.name);
     }

public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java

@@ -42,6 +42,9 @@ import java.util.*;
  *
  */
 public class GATKDoclet {
+    private final static String URL_ROOT_FOR_RELEASE_GATKDOCS = "http://www.broadinstitute.org/gsa/gatkdocs/release/";
+    private final static String URL_ROOT_FOR_STABLE_GATKDOCS = "http://iwww.broadinstitute.org/gsa/gatkdocs/stable/";
+    private final static String URL_ROOT_FOR_UNSTABLE_GATKDOCS = "http://iwww.broadinstitute.org/gsa/gatkdocs/unstable/";
     final protected static File SETTINGS_DIR = new File("settings/helpTemplates");
     final protected static File DESTINATION_DIR = new File("gatkdocs");
     final protected static Logger logger = Logger.getLogger(GATKDoclet.class);
@@ -89,6 +92,19 @@ public class GATKDoclet {
         return showHiddenFeatures;
     }
 
+    public static String htmlFilenameForClass(Class c) {
+        return c.getName().replace(".", "_") + ".html";
+    }
+
+    public static String helpLinksToGATKDocs(Class c) {
+        String classPath = htmlFilenameForClass(c);
+        StringBuilder b = new StringBuilder();
+        b.append("release  version: ").append(URL_ROOT_FOR_RELEASE_GATKDOCS).append(classPath).append("\n");
+        b.append("stable   version: ").append(URL_ROOT_FOR_STABLE_GATKDOCS).append(classPath).append("\n");
+        b.append("unstable version: ").append(URL_ROOT_FOR_UNSTABLE_GATKDOCS).append(classPath).append("\n");
+        return b.toString();
+    }
+
     public Set<GATKDocWorkUnit> workUnits() {
         TreeSet<GATKDocWorkUnit> m = new TreeSet<GATKDocWorkUnit>();
 
@@ -101,9 +117,9 @@ public class GATKDoclet {
 
             DocumentedGATKFeature feature = getFeatureForClassDoc(doc);
             DocumentedGATKFeatureHandler handler = createHandler(doc, feature);
-            if ( handler != null && handler.shouldBeProcessed(doc) ) {
+            if ( handler != null && handler.includeInDocs(doc) ) {
                 logger.info("Going to generate documentation for class " + doc);
-                String filename = handler.getDestinationFilename(doc);
+                String filename = handler.getDestinationFilename(doc, clazz);
                 GATKDocWorkUnit unit = new GATKDocWorkUnit(doc.name(),
                         filename, feature.groupName(),
                         feature, handler, doc, clazz,
@@ -220,7 +236,7 @@ public class GATKDoclet {
         Set<DocumentedGATKFeature> docFeatures = new HashSet<DocumentedGATKFeature>();
         List<Map<String, String>> data = new ArrayList<Map<String, String>>();
         for ( GATKDocWorkUnit workUnit : indexData ) {
-            data.add(workUnit.toMap());
+            data.add(workUnit.indexDataMap());
             docFeatures.add(workUnit.annotation);
         }
 

public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java

@@ -51,7 +51,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler {
     RootDoc rootDoc;
 
     @Override
-    public boolean shouldBeProcessed(ClassDoc doc) {
+    public boolean includeInDocs(ClassDoc doc) {
         return true;
 //        try {
 //            Class type = HelpUtils.getClassForDoc(doc);

public/java/test/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrationWalkersIntegrationTest.java

@@ -26,9 +26,9 @@ public class VariantRecalibrationWalkersIntegrationTest extends WalkerTest {
     }
 
     VRTest lowPass = new VRTest("phase1.projectConsensus.chr20.raw.snps.vcf",
-            "d33212a84368e821cbedecd4f59756d6",  // tranches
+            "0ddd1e0e483d2eaf56004615cea23ec7",  // tranches
-            "4652dca41222bebdf9d9fda343b2a835",  // recal file
+            "58780f63182e139fdbe17f6c18b5b774",  // recal file
-            "243a397a33a935fcaccd5deb6d16f0c0"); // cut VCF
+            "f67d844b6252a55452cf4167b77530b1"); // cut VCF
 
     @DataProvider(name = "VRTest")
     public Object[][] createData1() {

public/scala/qscript/org/broadinstitute/sting/queue/qscripts/GATKResourcesBundle.scala

@@ -134,6 +134,9 @@ class GATKResourcesBundle extends QScript {
     addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Unvalidated/AFR+EUR+ASN+1KG.dindel_august_release_merged_pilot1.20110126.sites.vcf",
       "1000G_indels_for_realignment", b37, true, false))
 
+    addResource(new Resource("/humgen/gsa-hpprojects/GATK/data/Comparisons/Validated/Mills_Devine_Indels_2011/ALL.wgs.indels_mills_devine_hg19_leftAligned_collapsed_double_hit.sites.vcf",
+      "indels_mills_devine", b37, true, true))
+    
     //
     // example call set for wiki tutorial
     //

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala

@@ -52,7 +52,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRu
 
     updateStatus(RunnerStatus.RUNNING)
     job.run()
-    updateStatus(RunnerStatus.FAILED)
+    updateStatus(RunnerStatus.DONE)
   }
 
   override def checkUnknownStatus() {}