From d31b176e159046e70880897b99777543d942ad17 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 20 Jul 2011 16:26:09 -0400 Subject: [PATCH 01/41] Removed GATK use of distributed parallelism framework. Moved distributed GATK prototype code into distributedutils, separating from threading package --- .../gatk/executive/LinearMicroScheduler.java | 2 +- .../sting/gatk/executive/MicroScheduler.java | 38 ------------------- .../ClosableReentrantLock.java | 2 +- .../FileBackedGenomeLocProcessingTracker.java | 2 +- .../GenomeLocProcessingTracker.java | 2 +- .../NoOpGenomeLocProcessingTracker.java | 2 +- .../ProcessingLoc.java | 2 +- .../SharedFileLock.java | 2 +- .../SharedFileThreadSafeLock.java | 2 +- ...haredMemoryGenomeLocProcessingTracker.java | 2 +- .../utils/distributedutils/package-info.java | 28 ++++++++++++++ .../GenomeLocProcessingTrackerUnitTest.java | 4 +- 12 files changed, 39 insertions(+), 49 deletions(-) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/ClosableReentrantLock.java (86%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/FileBackedGenomeLocProcessingTracker.java (98%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/GenomeLocProcessingTracker.java (99%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/NoOpGenomeLocProcessingTracker.java (93%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/ProcessingLoc.java (97%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/SharedFileLock.java (99%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/SharedFileThreadSafeLock.java (97%) rename public/java/src/org/broadinstitute/sting/utils/{threading => distributedutils}/SharedMemoryGenomeLocProcessingTracker.java (94%) create mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java rename public/java/test/org/broadinstitute/sting/utils/{threading => distributedutils}/GenomeLocProcessingTrackerUnitTest.java (99%) diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java index 9466fdf75..48fd73e0b 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/LinearMicroScheduler.java @@ -49,7 +49,7 @@ public class LinearMicroScheduler extends MicroScheduler { Accumulator accumulator = Accumulator.create(engine,walker); int counter = 0; - for (Shard shard : processingTracker.onlyOwned(shardStrategy, engine.getName())) { + for (Shard shard : shardStrategy ) { if ( shard == null ) // we ran out of shards that aren't owned break; diff --git a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java index 23e5769f1..e731b9864 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java +++ b/public/java/src/org/broadinstitute/sting/gatk/executive/MicroScheduler.java @@ -39,14 +39,10 @@ import org.broadinstitute.sting.gatk.traversals.*; import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import org.broadinstitute.sting.utils.exceptions.UserException; -import org.broadinstitute.sting.utils.threading.*; import javax.management.JMException; import javax.management.MBeanServer; import javax.management.ObjectName; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.PrintStream; import java.lang.management.ManagementFactory; import java.util.Collection; @@ -83,8 +79,6 @@ public abstract class MicroScheduler implements MicroSchedulerMBean { private final MBeanServer mBeanServer; private final ObjectName mBeanName; - protected GenomeLocProcessingTracker processingTracker; - /** * MicroScheduler factory function. Create a microscheduler appropriate for reducing the * selected walker. @@ -98,11 +92,6 @@ public abstract class MicroScheduler implements MicroSchedulerMBean { * @return The best-fit microscheduler. */ public static MicroScheduler create(GenomeAnalysisEngine engine, Walker walker, SAMDataSource reads, IndexedFastaSequenceFile reference, Collection rods, int nThreadsToUse) { - if (engine.getArguments().processingTrackerFile != null) { - if ( walker instanceof ReadWalker ) - throw new UserException.BadArgumentValue("C", String.format("Distributed GATK processing not enabled for read walkers")); - } - if (walker instanceof TreeReducible && nThreadsToUse > 1) { if(walker.isReduceByInterval()) throw new UserException.BadArgumentValue("nt", String.format("The analysis %s aggregates results by interval. Due to a current limitation of the GATK, analyses of this type do not currently support parallel execution. Please run your analysis without the -nt option.", engine.getWalkerName(walker.getClass()))); @@ -157,33 +146,6 @@ public abstract class MicroScheduler implements MicroSchedulerMBean { catch (JMException ex) { throw new ReviewedStingException("Unable to register microscheduler with JMX", ex); } - - // - // create the processing tracker - // - if ( engine.getArguments().processingTrackerFile != null ) { - logger.warn("Distributed GATK is an experimental engine feature, and is likely to not work correctly or reliably."); - if ( engine.getArguments().restartProcessingTracker && engine.getArguments().processingTrackerFile.exists() ) { - engine.getArguments().processingTrackerFile.delete(); - logger.info("Deleting ProcessingTracker file " + engine.getArguments().processingTrackerFile); - } - - PrintStream statusStream = null; - if ( engine.getArguments().processingTrackerStatusFile != null ) { - try { - statusStream = new PrintStream(new FileOutputStream(engine.getArguments().processingTrackerStatusFile)); - } catch ( FileNotFoundException e) { - throw new UserException.CouldNotCreateOutputFile(engine.getArguments().processingTrackerStatusFile, e); - } - } - - ClosableReentrantLock lock = new SharedFileThreadSafeLock(engine.getArguments().processingTrackerFile, engine.getArguments().processTrackerID); - processingTracker = new FileBackedGenomeLocProcessingTracker(engine.getArguments().processingTrackerFile, engine.getGenomeLocParser(), lock, statusStream) ; - logger.info("Creating ProcessingTracker using shared file " + engine.getArguments().processingTrackerFile + " process.id = " + engine.getName() + " CID = " + engine.getArguments().processTrackerID); - } else { - // create a NoOp version that doesn't do anything but say "yes" - processingTracker = new NoOpGenomeLocProcessingTracker(); - } } /** diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/ClosableReentrantLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java similarity index 86% rename from public/java/src/org/broadinstitute/sting/utils/threading/ClosableReentrantLock.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java index d16c19130..7f0c879e8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/ClosableReentrantLock.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import java.util.concurrent.locks.ReentrantLock; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/FileBackedGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java similarity index 98% rename from public/java/src/org/broadinstitute/sting/utils/threading/FileBackedGenomeLocProcessingTracker.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java index 3763ec67d..eac68cbdd 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/FileBackedGenomeLocProcessingTracker.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.GenomeLocParser; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java similarity index 99% rename from public/java/src/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTracker.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java index e97a73fb8..a7310743b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTracker.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import net.sf.picard.reference.IndexedFastaSequenceFile; import org.apache.log4j.Logger; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/NoOpGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java similarity index 93% rename from public/java/src/org/broadinstitute/sting/utils/threading/NoOpGenomeLocProcessingTracker.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java index ad2a6d31b..9807b6efa 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/NoOpGenomeLocProcessingTracker.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import java.util.Collection; import java.util.Collections; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/ProcessingLoc.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java similarity index 97% rename from public/java/src/org/broadinstitute/sting/utils/threading/ProcessingLoc.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java index ee2283dcf..0957ac1ae 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/ProcessingLoc.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.HasGenomeLocation; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/SharedFileLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java similarity index 99% rename from public/java/src/org/broadinstitute/sting/utils/threading/SharedFileLock.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java index 0f47da413..bda62b890 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/SharedFileLock.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import org.apache.log4j.Logger; import org.apache.lucene.store.*; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/SharedFileThreadSafeLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java similarity index 97% rename from public/java/src/org/broadinstitute/sting/utils/threading/SharedFileThreadSafeLock.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java index d70879a0a..49fc8208a 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/SharedFileThreadSafeLock.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; diff --git a/public/java/src/org/broadinstitute/sting/utils/threading/SharedMemoryGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java similarity index 94% rename from public/java/src/org/broadinstitute/sting/utils/threading/SharedMemoryGenomeLocProcessingTracker.java rename to public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java index 9bf8b58b1..0e62d0f63 100644 --- a/public/java/src/org/broadinstitute/sting/utils/threading/SharedMemoryGenomeLocProcessingTracker.java +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java @@ -1,4 +1,4 @@ -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; import java.io.PrintStream; import java.util.ArrayList; diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java new file mode 100644 index 000000000..033120dc5 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/** + * Utilities for prototype distributed GATK. No longer in use in the codebase + */ +package org.broadinstitute.sting.utils.distributedutils; diff --git a/public/java/test/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTrackerUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java similarity index 99% rename from public/java/test/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTrackerUnitTest.java rename to public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java index 78ab916db..139937e29 100644 --- a/public/java/test/org/broadinstitute/sting/utils/threading/GenomeLocProcessingTrackerUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java @@ -1,5 +1,5 @@ // our package -package org.broadinstitute.sting.utils.threading; +package org.broadinstitute.sting.utils.distributedutils; // the imports for unit testing. @@ -7,9 +7,9 @@ package org.broadinstitute.sting.utils.threading; import net.sf.picard.reference.IndexedFastaSequenceFile; import org.broadinstitute.sting.BaseTest; -import org.broadinstitute.sting.gatk.iterators.GenomeLocusIterator; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.GenomeLocParser; +import org.broadinstitute.sting.utils.distributedutils.*; import org.broadinstitute.sting.utils.exceptions.UserException; import org.testng.Assert; import org.testng.annotations.*; From 45c73ff0e59da0ddef03c4b7b500ebd037995c04 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Wed, 20 Jul 2011 17:16:33 -0400 Subject: [PATCH 02/41] Runs and emits an HTML document --- ivy.xml | 3 + .../sting/utils/help/GATKDoclet.java | 94 +++++++++++++++++++ .../help/ResourceBundleExtractorDoclet.java | 2 - 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java diff --git a/ivy.xml b/ivy.xml index 3f3d1c97f..a394aa6d7 100644 --- a/ivy.xml +++ b/ivy.xml @@ -30,6 +30,9 @@ + + + diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java new file mode 100644 index 000000000..d73c3d6fc --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import com.sun.javadoc.*; +import freemarker.template.Configuration; +import freemarker.template.DefaultObjectWrapper; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import org.broadinstitute.sting.gatk.walkers.Walker; +import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.classloader.JVMUtils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + +import java.io.*; +import java.util.*; + +/** + * + */ +public class GATKDoclet { + /** + * Extracts the contents of certain types of javadoc and adds them to an XML file. + * @param rootDoc The documentation root. + * @return Whether the JavaDoc run succeeded. + * @throws java.io.IOException if output can't be written. + */ + public static boolean start(RootDoc rootDoc) throws IOException { + /* ------------------------------------------------------------------- */ + /* You should do this ONLY ONCE in the whole application life-cycle: */ + + Configuration cfg = new Configuration(); + // Specify the data source where the template files come from. + // Here I set a file directory for it: + cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/")); + // Specify how templates will see the data-model. This is an advanced topic... + // but just use this: + cfg.setObjectWrapper(new DefaultObjectWrapper()); + + + /* ------------------------------------------------------------------- */ + /* You usually do these for many times in the application life-cycle: */ + + /* Create a data-model */ + // Create the root hash + Map root = new HashMap(); + // Put string ``user'' into the root + root.put("user", "Mark DePristo"); + + /* Get or create a template */ + Template temp = cfg.getTemplate("test.html"); + + /* Merge data-model with template */ + Writer out = new OutputStreamWriter(System.out); + try { + temp.process(root, out); + out.flush(); + } catch ( TemplateException e ) { + throw new ReviewedStingException("Failed to create GATK documentation", e); + } + return true; + } + + /** + * Validate the given options against options supported by this doclet. + * @param option Option to validate. + * @return Number of potential parameters; 0 if not supported. + */ + public static int optionLength(String option) { + return 0; + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 6ee12d42e..c0afcc338 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -186,8 +186,6 @@ public class ResourceBundleExtractorDoclet { * @return True if the JavaDoc is missing. False otherwise. */ private static boolean isRequiredJavadocMissing(ClassDoc classDoc) { - if(classDoc.containingPackage().name().contains("oneoffprojects")) - return false; return classDoc.commentText().length() == 0 || classDoc.commentText().contains("Created by IntelliJ"); } From 6fa17d86ae8fecc4a0bd1c9e54f82ee0c8866d3f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 21 Jul 2011 00:18:07 -0400 Subject: [PATCH 03/41] Completely hacked together version of a FreeMarker + javadoc + custom doclet walker documentation generator --- build.xml | 20 +++ .../sting/commandline/CommandLineProgram.java | 54 +++--- .../sting/commandline/ParsingEngine.java | 2 +- .../sting/utils/help/GATKDoclet.java | 167 +++++++++++++++--- .../help/ResourceBundleExtractorDoclet.java | 87 ++++++--- settings/helpTemplates/test.html | 43 +++++ 6 files changed, 300 insertions(+), 73 deletions(-) create mode 100644 settings/helpTemplates/test.html diff --git a/build.xml b/build.xml index fe4c7a3f4..b49c8fb4c 100644 --- a/build.xml +++ b/build.xml @@ -457,6 +457,26 @@ + + + + + + + + + + + + + + + diff --git a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java index aba4fc109..d88e7030e 100644 --- a/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java +++ b/public/java/src/org/broadinstitute/sting/commandline/CommandLineProgram.java @@ -43,7 +43,7 @@ import java.util.Locale; public abstract class CommandLineProgram { /** The command-line program and the arguments it returned. */ - protected ParsingEngine parser = null; + public ParsingEngine parser = null; /** the default log level */ @Argument(fullName = "logging_level", @@ -144,6 +144,11 @@ public abstract class CommandLineProgram { public static int result = -1; + @SuppressWarnings("unchecked") + public static void start(CommandLineProgram clp, String[] args) throws Exception { + start(clp, args, false); + } + /** * This function is called to start processing the command line, and kick * off the execute message of the program. @@ -153,7 +158,7 @@ public abstract class CommandLineProgram { * @throws Exception when an exception occurs */ @SuppressWarnings("unchecked") - public static void start(CommandLineProgram clp, String[] args) throws Exception { + public static void start(CommandLineProgram clp, String[] args, boolean dryRun) throws Exception { try { // setup our log layout @@ -180,8 +185,9 @@ public abstract class CommandLineProgram { // - InvalidArgument in case these arguments are specified by plugins. // - MissingRequiredArgument in case the user requested help. Handle that later, once we've // determined the full complement of arguments. - parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument, - ParsingEngine.ValidationType.InvalidArgument)); + if ( ! dryRun ) + parser.validate(EnumSet.of(ParsingEngine.ValidationType.MissingRequiredArgument, + ParsingEngine.ValidationType.InvalidArgument)); parser.loadArgumentsIntoObject(clp); // Initialize the logger using the loaded command line. @@ -195,36 +201,40 @@ public abstract class CommandLineProgram { if (isHelpPresent(parser)) printHelpAndExit(clp, parser); - parser.validate(); + if ( ! dryRun ) parser.validate(); } else { parser.parse(args); - if (isHelpPresent(parser)) - printHelpAndExit(clp, parser); + if ( ! dryRun ) { + if (isHelpPresent(parser)) + printHelpAndExit(clp, parser); - parser.validate(); + parser.validate(); + } parser.loadArgumentsIntoObject(clp); // Initialize the logger using the loaded command line. clp.setupLoggerLevel(layout); } - // if they specify a log location, output our data there - if (clp.toFile != null) { - FileAppender appender; - try { - appender = new FileAppender(layout, clp.toFile, false); - logger.addAppender(appender); - } catch (IOException e) { - throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists"); + if ( ! dryRun ) { + // if they specify a log location, output our data there + if (clp.toFile != null) { + FileAppender appender; + try { + appender = new FileAppender(layout, clp.toFile, false); + logger.addAppender(appender); + } catch (IOException e) { + throw new RuntimeException("Unable to re-route log output to " + clp.toFile + " make sure the destination exists"); + } } + + // regardless of what happens next, generate the header information + HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args); + + // call the execute + CommandLineProgram.result = clp.execute(); } - - // regardless of what happens next, generate the header information - HelpFormatter.generateHeaderInformation(clp.getApplicationDetails(), args); - - // call the execute - CommandLineProgram.result = clp.execute(); } catch (ArgumentException e) { clp.parser.printHelp(clp.getApplicationDetails()); diff --git a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java index 8423bb2f2..0dc18e6f9 100755 --- a/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java +++ b/public/java/src/org/broadinstitute/sting/commandline/ParsingEngine.java @@ -45,7 +45,7 @@ public class ParsingEngine { * A list of defined arguments against which command lines are matched. * Package protected for testing access. */ - ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions(); + public ArgumentDefinitions argumentDefinitions = new ArgumentDefinitions(); /** * A list of matches from defined arguments to command-line text. diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index d73c3d6fc..bf99be641 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -29,10 +29,14 @@ import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; +import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.CommandLineExecutable; +import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import scala.reflect.Print; import java.io.*; import java.util.*; @@ -40,7 +44,7 @@ import java.util.*; /** * */ -public class GATKDoclet { +public class GATKDoclet extends ResourceBundleExtractorDoclet { /** * Extracts the contents of certain types of javadoc and adds them to an XML file. * @param rootDoc The documentation root. @@ -48,47 +52,164 @@ public class GATKDoclet { * @throws java.io.IOException if output can't be written. */ public static boolean start(RootDoc rootDoc) throws IOException { - /* ------------------------------------------------------------------- */ - /* You should do this ONLY ONCE in the whole application life-cycle: */ + GATKDoclet doclet = new GATKDoclet(); + //PrintStream out = doclet.loadData(rootDoc, false); + doclet.processDocs(rootDoc, null); + return true; + } - Configuration cfg = new Configuration(); - // Specify the data source where the template files come from. - // Here I set a file directory for it: - cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/")); - // Specify how templates will see the data-model. This is an advanced topic... - // but just use this: - cfg.setObjectWrapper(new DefaultObjectWrapper()); + public static int optionLength(String option) { + return ResourceBundleExtractorDoclet.optionLength(option); + } + @Override + protected void processDocs(RootDoc rootDoc, PrintStream ignore) { + try { + /* ------------------------------------------------------------------- */ + /* You should do this ONLY ONCE in the whole application life-cycle: */ + Configuration cfg = new Configuration(); + // Specify the data source where the template files come from. + // Here I set a file directory for it: + cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/")); + // Specify how templates will see the data-model. This is an advanced topic... + // but just use this: + cfg.setObjectWrapper(new DefaultObjectWrapper()); + + for ( ClassDoc doc : rootDoc.classes() ) { + if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { + System.out.printf("Walker class %s%n", doc); + processWalkerDocs(cfg, doc); + //return; + } +// else +// System.out.printf("Excluding non-walker class %s%n", doc); + } + } catch ( FileNotFoundException e ) { + throw new RuntimeException(e); + } catch ( IOException e ) { + throw new RuntimeException(e); + } + } + + private void processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException { /* ------------------------------------------------------------------- */ /* You usually do these for many times in the application life-cycle: */ - /* Create a data-model */ // Create the root hash - Map root = new HashMap(); - // Put string ``user'' into the root - root.put("user", "Mark DePristo"); + Map root = buildWalkerDataModel(doc); /* Get or create a template */ Template temp = cfg.getTemplate("test.html"); /* Merge data-model with template */ - Writer out = new OutputStreamWriter(System.out); + Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/" + getClassName(doc).replace(".", "_") + ".html"))); try { temp.process(root, out); out.flush(); } catch ( TemplateException e ) { throw new ReviewedStingException("Failed to create GATK documentation", e); } - return true; } - /** - * Validate the given options against options supported by this doclet. - * @param option Option to validate. - * @return Number of potential parameters; 0 if not supported. - */ - public static int optionLength(String option) { - return 0; + + private Map buildWalkerDataModel(ClassDoc classdoc) { + Map root = new HashMap(); + + root.put("name", classdoc.name()); + + // Extract overrides from the doc tags. + StringBuilder summaryBuilder = new StringBuilder(); + for(Tag tag: classdoc.firstSentenceTags()) + summaryBuilder.append(tag.text()); + root.put("summary", summaryBuilder.toString()); + root.put("description", classdoc.commentText()); + + for(Tag tag: classdoc.tags()) { + root.put(tag.name(), tag.text()); + } + + ParsingEngine parsingEngine = createStandardGATKParsingEngine(); +// for (ArgumentDefinition argumentDefinition : parsingEngine.argumentDefinitions ) +// System.out.println(argumentDefinition); + + Map> args = new HashMap>(); + root.put("arguments", args); + args.put("required", new ArrayList()); + args.put("optional", new ArrayList()); + args.put("hidden", new ArrayList()); + args.put("depreciated", new ArrayList()); + try { + for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) { + String kind = "optional"; + if ( argumentSource.isRequired() ) kind = "required"; + else if ( argumentSource.isHidden() ) kind = "hidden"; + else if ( argumentSource.isDeprecated() ) kind = "depreciated"; + args.get(kind).add(argumentDataModel(argumentSource.createArgumentDefinitions().get(0))); + System.out.printf("Processing %s%n", argumentSource); + +// for(FieldDoc fieldDoc: classdoc.fields()) { +// //for ( AnnotationDesc desc : fieldDoc.annotations() ) { +// System.out.printf("AnnotationDesc %s%n", desc); +// if ( implementsInterface(desc.annotationType(), Argument.class, Output.class, Input.class) ) { +// (requiredAnnotation(desc) ? requiredArgs : optionalArgs).add(dataModelForArgument(desc)); +// System.out.printf("Processing %s%n", desc); +// } else { +// System.out.printf("Skipping %s%n", desc); +// } +// } +// } + } + } catch ( ClassNotFoundException e ) { + throw new RuntimeException(e); + } + + System.out.printf("Root is %s%n", root); + return root; + } + + protected String withDefault(String val, String def) { + return val == null ? def : val; + } + + protected Map argumentDataModel(ArgumentDefinition argumentDefinition) { + Map root = new HashMap(); + root.put("shortName", withDefault(argumentDefinition.shortName, "None provided")); + root.put("required", argumentDefinition.required); + root.put("fullName", withDefault(argumentDefinition.fullName, "None provided")); + root.put("argumentType", argumentDefinition.argumentType); + root.put("doc", withDefault(argumentDefinition.doc, "None provided")); + return root; + } + + protected ParsingEngine createStandardGATKParsingEngine() { + CommandLineProgram clp = new CommandLineGATK(); + try { + CommandLineProgram.start(clp, new String[]{}, true); + return clp.parser; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + protected Map dataModelForArgument(AnnotationDesc desc) { + Map root = new HashMap(); + root.put("shortName", "None provided"); + root.put("required", false); + root.put("fullName", "None provided"); + root.put("doc", "None provided"); + + for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { + root.put(keyvalue.element().name(), keyvalue.value().value()); + } + return root; + } + + protected boolean requiredAnnotation(AnnotationDesc desc) { + for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { + if ( keyvalue.element().name().equals("required") ) + return keyvalue.value().toString().equals("true"); + } + return false; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index c0afcc338..05408c2fa 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -30,12 +30,10 @@ import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import sun.tools.java.ClassNotFound; import java.io.*; -import java.util.HashSet; -import java.util.Properties; -import java.util.Scanner; -import java.util.Set; +import java.util.*; /** * Extracts certain types of javadoc (specifically package and class descriptions) and makes them available @@ -48,17 +46,19 @@ public class ResourceBundleExtractorDoclet { /** * Taglet for the particular version number. */ - private static final String VERSION_TAGLET_NAME = "version"; + protected static final String VERSION_TAGLET_NAME = "version"; /** * Maintains a collection of resources in memory as they're accumulated. */ - private static final Properties resourceText = new Properties(); + protected final Properties resourceText = new Properties(); /** * Maintains a collection of classes that should really be documented. */ - private static final Set undocumentedWalkers = new HashSet(); + protected final Set undocumentedWalkers = new HashSet(); + + protected String buildTimestamp = null, versionPrefix = null, versionSuffix = null, absoluteVersion = null; /** * Extracts the contents of certain types of javadoc and adds them to an XML file. @@ -67,13 +67,26 @@ public class ResourceBundleExtractorDoclet { * @throws IOException if output can't be written. */ public static boolean start(RootDoc rootDoc) throws IOException { + ResourceBundleExtractorDoclet doclet = new ResourceBundleExtractorDoclet(); + PrintStream out = doclet.loadData(rootDoc, true); + doclet.processDocs(rootDoc, out); + return true; + } + + protected PrintStream loadData(RootDoc rootDoc, boolean overwriteResourcesFile) { PrintStream out = System.out; - String buildTimestamp = null, versionPrefix = null, versionSuffix = null, absoluteVersion = null; for(String[] options: rootDoc.options()) { if(options[0].equals("-out")) { - loadExistingResourceFile(options[1], rootDoc); - out = new PrintStream(options[1]); + try { + loadExistingResourceFile(options[1], rootDoc); + if ( overwriteResourcesFile ) + out = new PrintStream(options[1]); + } catch ( FileNotFoundException e ) { + throw new RuntimeException(e); + } catch ( IOException e ) { + throw new RuntimeException(e); + } } if(options[0].equals("-build-timestamp")) buildTimestamp = options[1]; @@ -86,7 +99,10 @@ public class ResourceBundleExtractorDoclet { } resourceText.setProperty("build.timestamp",buildTimestamp); + return out; + } + protected void processDocs(RootDoc rootDoc, PrintStream out) { // Cache packages as we see them, since there's no direct way to iterate over packages. Set packages = new HashSet(); @@ -97,13 +113,19 @@ public class ResourceBundleExtractorDoclet { if(isRequiredJavadocMissing(currentClass) && isWalker(currentClass)) undocumentedWalkers.add(currentClass.name()); - renderHelpText(getClassName(currentClass),currentClass,versionPrefix,versionSuffix,absoluteVersion); + renderHelpText(getClassName(currentClass),currentClass); } for(PackageDoc currentPackage: packages) - renderHelpText(currentPackage.name(),currentPackage,versionPrefix,versionSuffix,absoluteVersion); + renderHelpText(currentPackage.name(),currentPackage); - resourceText.store(out,"Strings displayed by the Sting help system"); + try { + resourceText.store(out,"Strings displayed by the Sting help system"); + } catch ( FileNotFoundException e ) { + throw new RuntimeException(e); + } catch ( IOException e ) { + throw new RuntimeException(e); + } // ASCII codes for making text blink final String blink = "\u001B\u005B\u0035\u006D"; @@ -111,8 +133,6 @@ public class ResourceBundleExtractorDoclet { if(undocumentedWalkers.size() > 0) Utils.warnUser(String.format("The following walkers are currently undocumented: %s%s%s", blink, Utils.join(" ",undocumentedWalkers), reset)); - - return true; } /** @@ -137,7 +157,7 @@ public class ResourceBundleExtractorDoclet { * @throws IOException if there is an I/O-related error other than FileNotFoundException * while attempting to read the resource file. */ - private static void loadExistingResourceFile( String resourceFileName, RootDoc rootDoc ) throws IOException { + private void loadExistingResourceFile( String resourceFileName, RootDoc rootDoc ) throws IOException { try { BufferedReader resourceFile = new BufferedReader(new FileReader(resourceFileName)); try { @@ -157,10 +177,21 @@ public class ResourceBundleExtractorDoclet { * @param classDoc the type of the given class. * @return True if the class of the given name is a walker. False otherwise. */ - private static boolean isWalker(ClassDoc classDoc) { + protected static boolean isWalker(ClassDoc classDoc) { + return assignableToClass(classDoc, Walker.class, true); + } + + protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) { + for ( Class interfaceClass : interfaceClasses ) + if ( assignableToClass(classDoc, interfaceClass, false) ) + return true; + return false; + } + + protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { try { - Class type = Class.forName(getClassName(classDoc)); - return Walker.class.isAssignableFrom(type) && JVMUtils.isConcrete(type); + Class type = getClassForDoc(classDoc); + return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type)); } catch(Throwable t) { // Ignore errors. @@ -168,16 +199,20 @@ public class ResourceBundleExtractorDoclet { } } + protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException { + return Class.forName(getClassName(doc)); + } + /** * Reconstitute the class name from the given class JavaDoc object. - * @param classDoc the Javadoc model for the given class. + * @param doc the Javadoc model for the given class. * @return The (string) class name of the given class. */ - private static String getClassName(ClassDoc classDoc) { - PackageDoc containingPackage = classDoc.containingPackage(); + protected static String getClassName(ProgramElementDoc doc) { + PackageDoc containingPackage = doc.containingPackage(); return containingPackage.name().length() > 0 ? - String.format("%s.%s",containingPackage.name(),classDoc.name()) : - String.format("%s",classDoc.name()); + String.format("%s.%s",containingPackage.name(),doc.name()) : + String.format("%s",doc.name()); } /** @@ -193,10 +228,8 @@ public class ResourceBundleExtractorDoclet { * Renders all the help text required for a given name. * @param elementName element name to use as the key * @param element Doc element to process. - * @param versionPrefix Text to add to the start of the version string. - * @param versionSuffix Text to add to the end of the version string. */ - private static void renderHelpText(String elementName, Doc element, String versionPrefix, String versionSuffix, String absoluteVersion) { + private void renderHelpText(String elementName, Doc element) { // Extract overrides from the doc tags. String name = null; String version = null; diff --git a/settings/helpTemplates/test.html b/settings/helpTemplates/test.html new file mode 100644 index 000000000..6f6c222c4 --- /dev/null +++ b/settings/helpTemplates/test.html @@ -0,0 +1,43 @@ +<#macro argumentlist myargs> + + + + + + + <#list myargs as arg> + + + + + + <#-- + + --> + +
Short nameFull nameDescription
${arg.shortName}${arg.fullName}${arg.doc}
${arg.required}
+ + + + + ${name} documentation + + +

${name}

+

Summary

+ ${summary} +

Version

+ ${version!"unknown version"} + <#if author??> +

Author

+ ${author} + +

Description

+ ${description} +

Arguments

+

Required

<@argumentlist myargs=arguments.required/> +

Optional

<@argumentlist myargs=arguments.optional/> +

Hidden

<@argumentlist myargs=arguments.hidden/> +

Depreciated

<@argumentlist myargs=arguments.depreciated/> + + From e892489696ec01c2edda7da3ca9efe19a5d8b7f6 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 21 Jul 2011 15:20:34 -0400 Subject: [PATCH 04/41] V2 of the document system. Now uses GATKDoc class to organize documentation for arguments. Arguments now listed by feature (required, optional, hidden, etc) and link to detailed information about the argument in the html Lots of code moving between Class and ClassDoc objects. Should be refactored into a single static utility class. --- build.xml | 2 +- .../gatk/walkers/diffengine/DiffEngine.java | 1 + .../walkers/diffengine/DiffObjectsWalker.java | 79 ++++++++--- .../sting/utils/help/GATKDoc.java | 123 ++++++++++++++++++ .../sting/utils/help/GATKDoclet.java | 118 +++++++++++------ .../help/ResourceBundleExtractorDoclet.java | 12 +- settings/helpTemplates/test.html | 62 ++++++--- 7 files changed, 317 insertions(+), 80 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java diff --git a/build.xml b/build.xml index b49c8fb4c..dcaafe322 100644 --- a/build.xml +++ b/build.xml @@ -468,7 +468,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - additionalparam="-build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet"> + additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java index 89e20dad1..5f8f19892 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffEngine.java @@ -42,6 +42,7 @@ import java.util.*; * Date: 7/4/11 * Time: 12:51 PM * A generic engine for comparing tree-structured objects + * */ public class DiffEngine { final protected static Logger logger = Logger.getLogger(DiffEngine.class); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index fba6549fb..5cd99697c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -37,37 +37,86 @@ import java.io.PrintStream; import java.util.List; /** + * A generic engine for comparing tree-structured objects + * * Compares two record-oriented files, itemizing specific difference between equivalent * records in the two files. Reports both itemized and summarized differences. + * * @author Mark DePristo + * @since 7/4/11 * @version 0.1 */ @Requires(value={}) public class DiffObjectsWalker extends RodWalker { + /** + * Writes out a file of the DiffEngine format: + * + * http://www.broadinstitute.org/gsa/wiki/index.php/DiffEngine + */ @Output(doc="File to which results should be written",required=true) protected PrintStream out; - @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) - int MAX_OBJECTS_TO_READ = -1; - - @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) - int MAX_DIFFS = 0; - - @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) - int MAX_COUNT1_DIFFS = 0; - - @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) - int minCountForDiff = 1; - - @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) - boolean showItemizedDifferences = false; - + /** + * The master file against which we will compare test. This is one of the two required + * files to do the comparison. Conceptually master is the original file contained the expected + * results, but this doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="master", shortName="m", doc="Master file: expected results", required=true) File masterFile; + /** + * The test file against which we will compare to the master. This is one of the two required + * files to do the comparison. Conceptually test is the derived file from master, but this + * doesn't currently have an impact on the calculations, but might in the future. + */ @Argument(fullName="test", shortName="t", doc="Test file: new results to compare to the master file", required=true) File testFile; + /** + * The engine will read at most this number of objects from each of master and test files. This reduces + * the memory requirements for DiffObjects but does limit you to comparing at most this number of objects + */ + @Argument(fullName="maxObjectsToRead", shortName="motr", doc="Max. number of objects to read from the files. -1 [default] means unlimited", required=false) + int MAX_OBJECTS_TO_READ = -1; + + /** + * The max number of differences to display when summarizing. For example, if there are 10M differences, but + * maxDiffs is 10, then the comparison aborts after first ten summarized differences are shown. Note that + * the system shows differences sorted by frequency, so these 10 would be the most common between the two files. + * A value of 0 means show all possible differences. + */ + @Argument(fullName="maxDiffs", shortName="M", doc="Max. number of diffs to process", required=false) + int MAX_DIFFS = 0; + + /** + * The maximum number of singleton (occurs exactly once between the two files) to display when writing out + * the summary. Only applies if maxDiffs hasn't been exceeded. For example, if maxDiffs is 10 and maxCount1Diffs + * is 2 and there are 20 diffs with count > 1, then only 10 are shown, all of which have count above 1. + */ + @Argument(fullName="maxCount1Diffs", shortName="M1", doc="Max. number of diffs occuring exactly once in the file to process", required=false) + int MAX_COUNT1_DIFFS = 0; + + /** + * Only differences that occur more than minCountForDiff are displayed. For example, if minCountForDiff is 10, then + * a difference must occur at least 10 times between the two files to be shown. + */ + @Argument(fullName="minCountForDiff", shortName="MCFD", doc="Min number of observations for a records to display", required=false) + int minCountForDiff = 1; + + /** + * If provided, the system will write out the summarized, individual differences. May lead to enormous outputs, + * depending on how many differences are found. Note these are not sorted in any way, so if you have 10M + * common differences in the files, you will see 10M records, whereas the final summarize will just list the + * difference and its count of 10M. + */ + @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) + boolean showItemizedDifferences = false; + + @Argument(fullName="testEnum", doc="X", required=false) + TestEnum testEnum = TestEnum.ONE; + + public enum TestEnum { ONE, TWO }; + final DiffEngine diffEngine = new DiffEngine(); @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java new file mode 100644 index 000000000..0c575523e --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import org.broadinstitute.sting.utils.Utils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Created by IntelliJ IDEA. + * User: depristo + * Date: 7/21/11 + * Time: 8:51 AM + * + * Common documentation information about an GATK capability + */ +public class GATKDoc { + final DocType type; + final String name; + List synonyms; + String summary, fulltext; + final Map tags; + + public final static String NA_STRING = "None provided"; + + public enum DocType { + WALKER ("Walker"), + WALKER_ARG ("Walker argument"), + READ_FILTER ("Read filter"), + ENGINE_FEATURE ("Engine feature"); + + private final String name; + DocType(String name) { + this.name = name; + } + + }; + + public GATKDoc(DocType type, String name) { + this(type, name, new ArrayList(), NA_STRING, NA_STRING, new HashMap()); + } + + public GATKDoc(DocType type, String name, List synonyms, String summary, String fulltext, Map tags) { + this.type = type; + this.name = name; + this.synonyms = synonyms; + this.summary = summary; + this.fulltext = fulltext; + this.tags = tags; + } + + public Map toDataModel() { + Map model = new HashMap(); + model.put("type", type.name); + model.put("name", name); + model.put("synonyms", Utils.join(",", synonyms)); + model.put("summary", summary); + model.put("fulltext", fulltext); + model.putAll(tags); + return model; + } + + public DocType getType() { + return type; + } + + public String getName() { + return name; + } + + public List getSynonyms() { + return synonyms; + } + + public void addSynonym(String synonyms) { + this.synonyms.add(synonyms); + } + + public String getSummary() { + return summary; + } + + public void setSummary(String summary) { + this.summary = summary; + } + + public String getFulltext() { + return fulltext; + } + + public void setFulltext(String fulltext) { + this.fulltext = fulltext; + } + + public void addTag(String key, String value) { + this.tags.put(key, value); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index bf99be641..18b4266be 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -37,14 +37,18 @@ import org.broadinstitute.sting.utils.Utils; import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import scala.reflect.Print; +import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { + RootDoc root; + /** * Extracts the contents of certain types of javadoc and adds them to an XML file. * @param rootDoc The documentation root. @@ -64,6 +68,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { @Override protected void processDocs(RootDoc rootDoc, PrintStream ignore) { + // setup the global access to the root + root = rootDoc; + try { /* ------------------------------------------------------------------- */ /* You should do this ONLY ONCE in the whole application life-cycle: */ @@ -77,7 +84,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { cfg.setObjectWrapper(new DefaultObjectWrapper()); for ( ClassDoc doc : rootDoc.classes() ) { - if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { + if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { // && getClassName(doc).contains("UGCalcLikelihoods")) { System.out.printf("Walker class %s%n", doc); processWalkerDocs(cfg, doc); //return; @@ -135,30 +142,23 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Map> args = new HashMap>(); root.put("arguments", args); + args.put("all", new ArrayList()); args.put("required", new ArrayList()); args.put("optional", new ArrayList()); args.put("hidden", new ArrayList()); args.put("depreciated", new ArrayList()); try { for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) { + ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); + FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); + GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(argumentDataModel(argumentSource.createArgumentDefinitions().get(0))); + args.get(kind).add(doc.toDataModel()); + args.get("all").add(doc.toDataModel()); System.out.printf("Processing %s%n", argumentSource); - -// for(FieldDoc fieldDoc: classdoc.fields()) { -// //for ( AnnotationDesc desc : fieldDoc.annotations() ) { -// System.out.printf("AnnotationDesc %s%n", desc); -// if ( implementsInterface(desc.annotationType(), Argument.class, Output.class, Input.class) ) { -// (requiredAnnotation(desc) ? requiredArgs : optionalArgs).add(dataModelForArgument(desc)); -// System.out.printf("Processing %s%n", desc); -// } else { -// System.out.printf("Skipping %s%n", desc); -// } -// } -// } } } catch ( ClassNotFoundException e ) { throw new RuntimeException(e); @@ -168,19 +168,9 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - protected String withDefault(String val, String def) { - return val == null ? def : val; - } - - protected Map argumentDataModel(ArgumentDefinition argumentDefinition) { - Map root = new HashMap(); - root.put("shortName", withDefault(argumentDefinition.shortName, "None provided")); - root.put("required", argumentDefinition.required); - root.put("fullName", withDefault(argumentDefinition.fullName, "None provided")); - root.put("argumentType", argumentDefinition.argumentType); - root.put("doc", withDefault(argumentDefinition.doc, "None provided")); - return root; - } +// protected String withDefault(String val, String def) { +// return val == null ? def : val; +// } protected ParsingEngine createStandardGATKParsingEngine() { CommandLineProgram clp = new CommandLineGATK(); @@ -192,24 +182,66 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - protected Map dataModelForArgument(AnnotationDesc desc) { - Map root = new HashMap(); - root.put("shortName", "None provided"); - root.put("required", false); - root.put("fullName", "None provided"); - root.put("doc", "None provided"); - - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - root.put(keyvalue.element().name(), keyvalue.value().value()); - } - return root; + private FieldDoc getFieldDoc(ClassDoc classDoc, String name) { + return getFieldDoc(classDoc, name, true); } - protected boolean requiredAnnotation(AnnotationDesc desc) { - for ( AnnotationDesc.ElementValuePair keyvalue : desc.elementValues() ) { - if ( keyvalue.element().name().equals("required") ) - return keyvalue.value().toString().equals("true"); + private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { + System.out.printf("Looking for %s in %s%n", name, classDoc.name()); + for ( FieldDoc fieldDoc : classDoc.fields(false) ) { + System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); + if ( fieldDoc.name().equals(name) ) + return fieldDoc; + + Field field = getFieldForFieldDoc(fieldDoc); + if ( field.isAnnotationPresent(ArgumentCollection.class) ) { + ClassDoc typeDoc = root.classNamed(fieldDoc.type().qualifiedTypeName()); + if ( typeDoc == null ) + throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); + else { + FieldDoc result = getFieldDoc(typeDoc, name, false); + if ( result != null ) + return result; + // else keep searching + } + } } - return false; + + // if we didn't find it here, wander up to the superclass to find the field + if ( classDoc.superclass() != null ) { + return getFieldDoc(classDoc.superclass(), name, false); + } + + if ( primary ) + throw new RuntimeException("No field found for expected field " + name); + else + return null; + } + + protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { + final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; + GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); + + if ( def.fullName != null && def.shortName != null) + doc.addSynonym("-" + def.shortName); + + doc.addTag("required", def.required ? "yes" : "no"); + doc.addTag("type", def.argumentType.getSimpleName()); + if ( def.doc != null ) doc.setSummary(def.doc); + + List attributes = new ArrayList(); + attributes.add(def.ioType.annotationClass.getSimpleName()); + if ( def.required ) attributes.add("required"); + if ( def.isFlag ) attributes.add("flag"); + if ( def.isHidden ) attributes.add("hidden"); + doc.addTag("attributes", Utils.join(",", attributes)); + + // todo -- need depreciated value + + doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); + + doc.setFulltext(fieldDoc.commentText()); + + return doc; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 05408c2fa..4f4c411a3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -33,6 +33,7 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import sun.tools.java.ClassNotFound; import java.io.*; +import java.lang.reflect.Field; import java.util.*; /** @@ -191,7 +192,7 @@ public class ResourceBundleExtractorDoclet { protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { try { Class type = getClassForDoc(classDoc); - return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type)); + return lhsClass.isAssignableFrom(type) && (! requireConcrete || JVMUtils.isConcrete(type)); } catch(Throwable t) { // Ignore errors. @@ -203,6 +204,15 @@ public class ResourceBundleExtractorDoclet { return Class.forName(getClassName(doc)); } + protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { + try { + Class clazz = getClassForDoc(fieldDoc.containingClass()); + return JVMUtils.findField(clazz, fieldDoc.name()); + } catch ( ClassNotFoundException e ) { + throw new RuntimeException(e); + } + } + /** * Reconstitute the class name from the given class JavaDoc object. * @param doc the Javadoc model for the given class. diff --git a/settings/helpTemplates/test.html b/settings/helpTemplates/test.html index 6f6c222c4..f9bfc2b92 100644 --- a/settings/helpTemplates/test.html +++ b/settings/helpTemplates/test.html @@ -1,23 +1,37 @@ -<#macro argumentlist myargs> - - - - - - - <#list myargs as arg> +<#macro argumentlist name myargs> + <#if myargs?size != 0> +

${name}

+
Short nameFull nameDescription
- - - + + + + - <#-- - - --> - -
${arg.shortName}${arg.fullName}${arg.doc}NameSynonymsTypeSummary
${arg.required}
+ <#list myargs as arg> + + ${arg.name} + ${arg.synonyms} + ${arg.type} + ${arg.summary} + + <#-- + ${arg.required} + --> + + + +<#macro argumentDetails arg> +

${arg.name} / ${arg.synonyms}

+ Summary: ${arg.summary}
+ Attributes: ${arg.attributes}
+ Type: ${arg.type}
+ Options: ${arg.options}
+ Details: ${arg.fulltext}
+ + ${name} documentation @@ -34,10 +48,18 @@

Description

${description} + <#-- Create the argument summary -->

Arguments

-

Required

<@argumentlist myargs=arguments.required/> -

Optional

<@argumentlist myargs=arguments.optional/> -

Hidden

<@argumentlist myargs=arguments.hidden/> -

Depreciated

<@argumentlist myargs=arguments.depreciated/> + <@argumentlist name="Required" myargs=arguments.required/> + <@argumentlist name="Optional" myargs=arguments.optional/> + <@argumentlist name="Hidden" myargs=arguments.hidden/> + <@argumentlist name="Depreciated" myargs=arguments.depreciated/> + + <#-- Create the argument details --> +

Argument details

+ <#list arguments.all as arg> + <@argumentDetails arg=arg/> + + From 81d0cab27eff703b473a994b7703630e04f6da87 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Thu, 21 Jul 2011 16:01:54 -0400 Subject: [PATCH 05/41] Walker index html now emited. --- .../walkers/diffengine/DiffObjectsWalker.java | 74 ++++++++++++++++++- .../sting/utils/help/GATKDoclet.java | 53 +++++++++---- .../{test.html => walker.template.html} | 28 ++++--- 3 files changed, 123 insertions(+), 32 deletions(-) rename settings/helpTemplates/{test.html => walker.template.html} (69%) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index 5cd99697c..899c3671c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.walkers.diffengine; import org.broadinstitute.sting.commandline.Argument; +import org.broadinstitute.sting.commandline.Hidden; import org.broadinstitute.sting.commandline.Output; import org.broadinstitute.sting.gatk.contexts.AlignmentContext; import org.broadinstitute.sting.gatk.contexts.ReferenceContext; @@ -38,9 +39,79 @@ import java.util.List; /** * A generic engine for comparing tree-structured objects - * + *

* Compares two record-oriented files, itemizing specific difference between equivalent * records in the two files. Reports both itemized and summarized differences. + *

+ * What are the summarized differences and the DiffObjectsWalker + *

+ * The GATK contains a summarizing difference engine that compares hierarchical data structures to emit: + *

    + *
  • A list of specific differences between the two data structures. This is similar to saying the value in field A in record 1 in file F differences from the value in field A in record 1 in file G. + *
  • A summarized list of differences ordered by frequency of the difference. This output is similar to saying field A in 50 records in files F and G differed. + *
+ * + *

+ * The GATK contains a private walker DiffObjects that allows you access to the DiffEngine capabilities on the command line. Simply provide the walker with the master and test files and it will emit summarized differences for you. + * + *

+ * Why? + *

+ * The reason for this system is that it allows you to compare two structured files -- such as BAMs and VCFs -- for common differences among them. This is primarily useful in regression testing or optimization, where you want to ensure that the differences are those that you expect and not any others. + * + *

Understanding the output + *

The DiffEngine system compares to two hierarchical data structures for specific differences in the values of named + * nodes. Suppose I have two trees: + * 

+ *     Tree1=(A=1 B=(C=2 D=3))
+ *     Tree2=(A=1 B=(C=3 D=3 E=4))
+ *     Tree3=(A=1 B=(C=4 D=3 E=4))
+ *
+ *

+ * where every node in the tree is named, or is a raw value (here all leaf values are integers). The DiffEngine + * traverses these data structures by name, identifies equivalent nodes by fully qualified names + * (Tree1.A is distinct from Tree2.A, and determines where their values are equal (Tree1.A=1, Tree2.A=1, so they are). + * These itemized differences are listed as: + * 

+ *     Tree1.B.C=2 != Tree2.B.C=3
+ *     Tree1.B.C=2 != Tree3.B.C=4
+ *     Tree2.B.C=3 != Tree3.B.C=4
+ *     Tree1.B.E=MISSING != Tree2.B.E=4
+ *
+ *

+ * This conceptually very similar to the output of the unix command line tool diff. What's nice about DiffEngine though + * is that it computes similarity among the itemized differences and displays the count of differences names + * in the system. In the above example, the field C is not equal three times, while the missing E in Tree1 occurs + * only once. So the summary is: + * + * 

+ *     *.B.C : 3
+ *     *.B.E : 1
+ *
+ *

where the * operator indicates that any named field matches. This output is sorted by counts, and provides an + * immediate picture of the commonly occurring differences among the files. + *

+ * Below is a detailed example of two VCF fields that differ because of a bug in the AC, AF, and AN counting routines, + * detected by the integrationtest integration (more below). You can see that in the although there are many specific + * instances of these differences between the two files, the summarized differences provide an immediate picture that + * the AC, AF, and AN fields are the major causes of the differences. + *

+ * 

+   [testng] path                                                             count
+   [testng] *.*.*.AC                                                         6
+   [testng] *.*.*.AF                                                         6
+   [testng] *.*.*.AN                                                         6
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000000.AC  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000000.AF  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000000.AN  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000117.AC  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000117.AF  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000117.AN  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000211.AC  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000211.AF  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000211.AN  1
+   [testng] 64b991fd3850f83614518f7d71f0532f.integrationtest.20:10000598.AC  1
+
* * @author Mark DePristo * @since 7/4/11 @@ -112,6 +183,7 @@ public class DiffObjectsWalker extends RodWalker { @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) boolean showItemizedDifferences = false; + @Hidden @Argument(fullName="testEnum", doc="X", required=false) TestEnum testEnum = TestEnum.ONE; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 18b4266be..b6488cff9 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -47,7 +47,7 @@ import java.util.*; * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { - RootDoc root; + RootDoc rootDoc; /** * Extracts the contents of certain types of javadoc and adds them to an XML file. @@ -69,7 +69,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { @Override protected void processDocs(RootDoc rootDoc, PrintStream ignore) { // setup the global access to the root - root = rootDoc; + this.rootDoc = rootDoc; try { /* ------------------------------------------------------------------- */ @@ -83,15 +83,14 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { // but just use this: cfg.setObjectWrapper(new DefaultObjectWrapper()); + List> indexData = new ArrayList>(); for ( ClassDoc doc : rootDoc.classes() ) { if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { // && getClassName(doc).contains("UGCalcLikelihoods")) { System.out.printf("Walker class %s%n", doc); - processWalkerDocs(cfg, doc); - //return; + indexData.add(processWalkerDocs(cfg, doc)); } -// else -// System.out.printf("Excluding non-walker class %s%n", doc); } + processWalkerIndex(indexData,cfg); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -99,19 +98,15 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - private void processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException { - /* ------------------------------------------------------------------- */ - /* You usually do these for many times in the application life-cycle: */ - - // Create the root hash - Map root = buildWalkerDataModel(doc); - + private void processWalkerIndex(List> indexData, Configuration cfg) throws IOException { /* Get or create a template */ - Template temp = cfg.getTemplate("test.html"); + Template temp = cfg.getTemplate("walker.index.template.html"); /* Merge data-model with template */ - Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/" + getClassName(doc).replace(".", "_") + ".html"))); + Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); try { + Map root = new HashMap(); + root.put("walkers", indexData); temp.process(root, out); out.flush(); } catch ( TemplateException e ) { @@ -119,6 +114,32 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } + private Map processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException { + // Create the root hash + Map root = buildWalkerDataModel(doc); + + /* Get or create a template */ + Template temp = cfg.getTemplate("walker.template.html"); + + /* Merge data-model with template */ + File outputFile = new File(getClassName(doc).replace(".", "_") + ".html"); + File outputPath = new File("testdoc/" + outputFile); + try { + Writer out = new OutputStreamWriter(new FileOutputStream(outputPath)); + temp.process(root, out); + out.flush(); + } catch ( TemplateException e ) { + throw new ReviewedStingException("Failed to create GATK documentation", e); + } + + // add index data + Map indexData = new HashMap(); + indexData.put("filename", outputFile.toString()); + indexData.put("name", doc.name()); + indexData.put("summary", root.get("summary")); + return indexData; + } + private Map buildWalkerDataModel(ClassDoc classdoc) { Map root = new HashMap(); @@ -195,7 +216,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Field field = getFieldForFieldDoc(fieldDoc); if ( field.isAnnotationPresent(ArgumentCollection.class) ) { - ClassDoc typeDoc = root.classNamed(fieldDoc.type().qualifiedTypeName()); + ClassDoc typeDoc = this.rootDoc.classNamed(fieldDoc.type().qualifiedTypeName()); if ( typeDoc == null ) throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); else { diff --git a/settings/helpTemplates/test.html b/settings/helpTemplates/walker.template.html similarity index 69% rename from settings/helpTemplates/test.html rename to settings/helpTemplates/walker.template.html index f9bfc2b92..ecf24a56e 100644 --- a/settings/helpTemplates/test.html +++ b/settings/helpTemplates/walker.template.html @@ -1,13 +1,6 @@ <#macro argumentlist name myargs> <#if myargs?size != 0> -

${name}

- - - - - - - + <#list myargs as arg> @@ -19,7 +12,6 @@ --> -
NameSynonymsTypeSummary
${name}
${arg.name}${arg.required}
@@ -40,8 +32,6 @@

${name}

Summary

${summary} -

Version

- ${version!"unknown version"} <#if author??>

Author

${author} @@ -50,10 +40,18 @@ ${description} <#-- Create the argument summary -->

Arguments

- <@argumentlist name="Required" myargs=arguments.required/> - <@argumentlist name="Optional" myargs=arguments.optional/> - <@argumentlist name="Hidden" myargs=arguments.hidden/> - <@argumentlist name="Depreciated" myargs=arguments.depreciated/> + + + + + + + + <@argumentlist name="Required" myargs=arguments.required/> + <@argumentlist name="Optional" myargs=arguments.optional/> + <@argumentlist name="Hidden" myargs=arguments.hidden/> + <@argumentlist name="Depreciated" myargs=arguments.depreciated/> +
NameSynonymsTypeSummary
<#-- Create the argument details -->

Argument details

From e922ebc29dc224236caabe5eadd01897af9cfe9e Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 09:13:33 -0400 Subject: [PATCH 07/41] Missed this template for walker index. --- .../helpTemplates/walker.index.template.html | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 settings/helpTemplates/walker.index.template.html diff --git a/settings/helpTemplates/walker.index.template.html b/settings/helpTemplates/walker.index.template.html new file mode 100644 index 000000000..5d4cd84d4 --- /dev/null +++ b/settings/helpTemplates/walker.index.template.html @@ -0,0 +1,21 @@ + + + GATK walker index + + +

GATK walker index

+ + + + + + <#list walkers as walker> + + + + + +
NameSummary
${walker.name}${walker.summary}
+ + + From 172b35372babc6f681e733924d1308bd943c4669 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 09:20:32 -0400 Subject: [PATCH 08/41] Moved all of the distributed GATK code to archive. --- .../ClosableReentrantLock.java | 16 - .../FileBackedGenomeLocProcessingTracker.java | 114 ---- .../GenomeLocProcessingTracker.java | 486 ------------------ .../NoOpGenomeLocProcessingTracker.java | 26 - .../utils/distributedutils/ProcessingLoc.java | 71 --- .../distributedutils/SharedFileLock.java | 171 ------ .../SharedFileThreadSafeLock.java | 75 --- ...haredMemoryGenomeLocProcessingTracker.java | 34 -- .../utils/distributedutils/package-info.java | 28 - .../GenomeLocProcessingTrackerUnitTest.java | 402 --------------- 10 files changed, 1423 deletions(-) delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java delete mode 100644 public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java deleted file mode 100644 index 7f0c879e8..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/ClosableReentrantLock.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import java.util.concurrent.locks.ReentrantLock; - -/** - * Created by IntelliJ IDEA. - * User: depristo - * Date: 1/19/11 - * Time: 9:50 AM - * - * Simple extension of a ReentrantLock that supports a close method. - */ -public class ClosableReentrantLock extends ReentrantLock { - public boolean ownsLock() { return super.isHeldByCurrentThread(); } - public void close() {} -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java deleted file mode 100644 index eac68cbdd..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/FileBackedGenomeLocProcessingTracker.java +++ /dev/null @@ -1,114 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import org.apache.log4j.Logger; -import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.UserException; - -import java.io.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Keeps a copy of the processing locks in a file - */ -public class FileBackedGenomeLocProcessingTracker extends GenomeLocProcessingTracker { - private static final Logger logger = Logger.getLogger(FileBackedGenomeLocProcessingTracker.class); - private static final boolean DEBUG = false; - private static final String READ_MODE = "r"; - private static final String WRITE_MODE = "rws"; - - private final File sharedFile; - private final GenomeLocParser parser; - private long lastReadPosition = 0; - - public FileBackedGenomeLocProcessingTracker(File sharedFile, GenomeLocParser parser, ClosableReentrantLock lock, PrintStream status) { - super(lock, status); - - this.sharedFile = sharedFile; - this.parser = parser; - } - - private RandomAccessFile openFile(String mode) { - try { - return new RandomAccessFile(sharedFile, mode); - } catch (FileNotFoundException e) { - throw new UserException.CouldNotCreateOutputFile(sharedFile, e); - } - } - - private void closeFile(RandomAccessFile raFile) { - try { - if ( raFile != null ) raFile.close(); - } catch (IOException e) { - throw new UserException.CouldNotCreateOutputFile(sharedFile, e); - } - } - - @Override - protected List readNewLocs() { - List newPLocs = new ArrayList(); // todo -- gratitous object creation - - if ( sharedFile.exists() ) { - RandomAccessFile raFile = null; - try { - raFile = openFile(READ_MODE); - //logger.warn(String.format("Reading new locs at: file.length=%d last=%d", raFile.length(), lastReadPosition)); - if ( raFile.length() > lastReadPosition ) { - raFile.seek(lastReadPosition); - - int counter = 0; - String line = raFile.readLine(); // Read another line - while ( line != null ) { - String[] parts = line.split(" "); - if ( parts.length != 2 ) throw new ReviewedStingException("BUG: bad sharedFile line '" + line + "' at " + raFile.getFilePointer()); - ProcessingLoc ploc = new ProcessingLoc(parser.parseGenomeLoc(parts[0]), parts[1]); - //logger.warn(" Read " + ploc); - newPLocs.add(ploc); - line = raFile.readLine(); - counter++; - } - lastReadPosition = raFile.getFilePointer(); - if ( DEBUG ) logger.warn(String.format("Read %s locs from file, current pos is %d, # read new locs is %d", - counter, lastReadPosition, newPLocs.size())); - } - } catch (FileNotFoundException e) { - throw new UserException.CouldNotReadInputFile(sharedFile, e); - } catch (IOException e) { - throw new ReviewedStingException("Couldn't read sharedFile " + sharedFile, e); - } finally { - closeFile(raFile); - } - } - - return newPLocs; - } - - @Override - protected void registerNewLocs(Collection plocs) { - RandomAccessFile raFile = null; - - try { - raFile = openFile(WRITE_MODE); - long startPos = raFile.getFilePointer(); - raFile.seek(raFile.length()); - StringBuffer bytes = new StringBuffer(); - for ( ProcessingLoc ploc : plocs ) { - String packet = String.format("%s %s%n", ploc.getLocation(), ploc.getOwner()); - bytes.append(packet); - if ( DEBUG ) logger.warn(String.format("Wrote loc %s to file: %d + %d bytes ending at %d", ploc, startPos, packet.length(), raFile.getFilePointer())); - } - raFile.write(bytes.toString().getBytes()); - //raFile.getChannel().force(true); - } catch (FileNotFoundException e) { - throw new UserException.CouldNotCreateOutputFile(sharedFile, e); - } catch (IOException e) { - throw new UserException.CouldNotCreateOutputFile(sharedFile, e); - } finally { - closeFile(raFile); - } - } -} - - diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java deleted file mode 100644 index a7310743b..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTracker.java +++ /dev/null @@ -1,486 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import net.sf.picard.reference.IndexedFastaSequenceFile; -import org.apache.log4j.Logger; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.HasGenomeLocation; -import org.broadinstitute.sting.utils.SimpleTimer; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.UserException; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.PrintStream; -import java.text.SimpleDateFormat; -import java.util.*; - -/** - * Abstract base class to coordinating data processing by a collecting for processes / threads. - * - * Conceptually, the genome is viewed as a collection of non-overlapping genome location: - * - * chr1:1-10 - * chr1:11-20 - * chr1:21-30 - * etc. - * - * This class, and it's concrete derived classes, provide the ability to claim individual locations - * as "mine", and exclude other processes / threads from processing them. At the lowest-level this - * is implemented by the claimOwnership(loc, name) function, that returns true if loc free (unclaimed) - * and makes name the owner of loc. High-level, and more efficient operations provide claiming - * iterators over streams of objects implementing the HasGenomeLocation interface, so that you can - * write code that looks like: - * - * for ( GenomeLoc ownedLoc : onlyOwned(allLocsToProcess.iterator) ) { - * doSomeWork(ownedLoc) - * - * Much of the code in this class is actually surrounding debugging and performance metrics code. - * The actual synchronization code is separated out into the ClosableReentrantLock() system - * and the two abstract functions: - * - * protected abstract void registerNewLocs(Collection plocs); - * protected abstract Collection readNewLocs(); - * - * That maintain the state of the tracker. - * - * That is, the ProcessingTracker is made of two components: a thread / process locking system and - * a subclass that implements the methods to record new claimed state changes and to read out updates - * that may have occurred by another thread or process. - * - * NOTE: this class assumes that all threads / processes are working with the same set of potential - * GenomeLocs to own. Claiming chr1:1-10 and then chr1:5-6 is allowed by the system. Basically, - * you only can stake claim to GenomeLocs that are .equal(). - */ -public abstract class GenomeLocProcessingTracker { - private final static Logger logger = Logger.getLogger(FileBackedGenomeLocProcessingTracker.class); - private final static SimpleDateFormat STATUS_FORMAT = new SimpleDateFormat("HH:mm:ss,SSS"); - private final static int DEFAULT_OWNERSHIP_ITERATOR_SIZE = 1; - - /** - * Useful state strings for printing status - */ - private final static String GOING_FOR_LOCK = "going_for_lock"; - private final static String RELEASING_LOCK = "releasing_lock"; - private final static String HAVE_LOCK = "have_lock"; - private final static String RUNNING = "running"; - - /** - * A map, for efficiency, that allows quick lookup of the processing loc for a - * given GenomeLoc. The map points from loc -> loc / owner as a ProcessingLoc - */ - private final Map processingLocs; - - /** - * The locking object used to protect data from simulatanous access by multiple - * threads or processes. - */ - private final ClosableReentrantLock lock; - - /** A stream for writing status messages. Can be null if we aren't writing status */ - private final PrintStream status; - - // - // Timers for recording performance information - // Note -- these cannot be used because this class isn't thread safe, and neither are the - // timers, so they result in invalid operations w.r.t. the SimpleTimer contract - // -// protected final SimpleTimer writeTimer = new SimpleTimer("writeTimer"); -// protected final SimpleTimer readTimer = new SimpleTimer("readTimer"); -// protected final SimpleTimer lockWaitTimer = new SimpleTimer("lockWaitTimer"); - protected final SimpleTimer timer = new SimpleTimer(); - protected long nLocks = 0, nWrites = 0, nReads = 0; - - // -------------------------------------------------------------------------------- - // - // Creating ProcessingTrackers - // - // -------------------------------------------------------------------------------- - public GenomeLocProcessingTracker(ClosableReentrantLock lock, PrintStream status) { - this.processingLocs = new HashMap(); - this.status = status; - this.lock = lock; - printStatusHeader(); - } - - // -------------------------------------------------------------------------------- - // - // Code to override to change the dynamics of the the GenomeLocProcessingTracker - // - // -------------------------------------------------------------------------------- - - protected void close() { - lock.close(); - if ( status != null ) status.close(); - } - - /** - * Takes a collection of newly claimed (i.e., previous unclaimed) genome locs - * and the name of their owner and "registers" this data in some persistent way that's - * visible to all threads / processes communicating via this GenomeLocProcessingTracker. - * - * Could be a in-memory data structure (a list) if we are restricting ourselves to intra-memory - * parallelism, a locked file on a shared file system, or a server we communicate with. - * - * @param plocs - */ - protected abstract void registerNewLocs(Collection plocs); - - /** - * The inverse of the registerNewLocs() function. Looks at the persistent data store - * shared by all threads / processes and returns the ones that have appeared since the last - * call to readNewLocs(). Note that we expect the pair of registerNewLocs and readNewLocs to - * include everything, even locs registered by this thread / process. For example: - * - * readNewLocs() => List() - * registerNewLocs(List(x, y,)) => void - * readNewLocs() => List(x,y)) - * - * even for this thread or process. - * @return - */ - protected abstract Collection readNewLocs(); - - - // -------------------------------------------------------------------------------- - // - // Code to claim intervals for processing and query for their ownership - // - // -------------------------------------------------------------------------------- - - /** - * Queries the current database if a location is owned. Does not guarantee that the - * loc can be owned in a future call, though. - * - * @param loc - * @return - */ - public final boolean locIsOwned(GenomeLoc loc, String id) { - return findOwner(loc, id) != null; - } - - /** - * The workhorse routine. Attempt to claim processing ownership of loc, with my name. - * This is an atomic operation -- other threads / processes will wait until this function - * returns. The return result is the ProcessingLoc object describing who owns this - * location. If the location isn't already claimed and we now own the location, the pl owner - * will be myName. Otherwise, the name of the owner can found in the pl. - * - * @param loc - * @param myName - * @return - */ - public final ProcessingLoc claimOwnership(final GenomeLoc loc, final String myName) { - // processingLocs is a shared memory synchronized object, and this - // method is synchronized, so we can just do our processing - return new WithLock(myName) { - public ProcessingLoc doBody() { - ProcessingLoc owner = findOwner(loc, myName); - if ( owner == null ) { // we are unowned - owner = new ProcessingLoc(loc, myName); - registerNewLocsWithTimers(Arrays.asList(owner), myName); - } - return owner; - } - }.run(); - } - - - // -------------------------------------------------------------------------------- - // - // High-level iterator-style interface to claiming ownership - // - // -------------------------------------------------------------------------------- - - /** - * A higher-level, and more efficient, interface to obtain the next location we own. Takes an - * iterator producing objects that support the getLocation() interface, and returns the next - * object in that stream that we can claim ownership of. Returns null if we run out of elements - * during the iteration. - * - * Can be more efficiently implemented in subclasses to avoid multiple unlocking - * - * @param iterator - * @param myName - * @return - */ - public final T claimOwnershipOfNextAvailable(Iterator iterator, String myName) { - OwnershipIterator myIt = new OwnershipIterator(iterator, myName, 1); - return myIt.next(); - } - - public final Iterable onlyOwned(Iterator iterator, String myName) { - return new OwnershipIterator(iterator, myName); - } - - private final class OwnershipIterator implements Iterator, Iterable { - private final Iterator subit; - private final String myName; - private final Queue cache; - private final int cacheSize; - - public OwnershipIterator(Iterator subit, String myName) { - this(subit, myName, DEFAULT_OWNERSHIP_ITERATOR_SIZE); - } - - public OwnershipIterator(Iterator subit, String myName, int cacheSize) { - this.subit = subit; - this.myName = myName; - cache = new LinkedList(); - this.cacheSize = cacheSize; - } - - /** - * Will return true for all elements of subit, even if we can't get ownership of some of the future - * elements and so will return null there - * @return - */ - public final boolean hasNext() { - return cache.peek() != null || subit.hasNext(); - } - - /** - * High performance iterator that only locks and unlocks once per claimed object found. Avoids - * locking / unlocking for each query - * - * @return an object of type T owned by this thread, or null if none of the remaining object could be claimed - */ - public final T next() { - if ( cache.peek() != null) - return cache.poll(); - else { - // cache is empty, we need to fill up the cache and return the first element of the queue - return new WithLock(myName) { - public T doBody() { - // read once the database of owners at the start - updateAndGetProcessingLocs(myName); - - boolean done = false; - Queue pwns = new LinkedList(); // ;-) - while ( !done && cache.size() < cacheSize && subit.hasNext() ) { - final T elt = subit.next(); - GenomeLoc loc = elt.getLocation(); - - ProcessingLoc owner = processingLocs.get(loc); - - if ( owner == null ) { // we are unowned - owner = new ProcessingLoc(loc, myName); - pwns.offer(owner); - if ( ! cache.offer(elt) ) throw new ReviewedStingException("Cache offer unexpectedly failed"); - if ( GenomeLoc.isUnmapped(loc) ) done = true; - } - // if not, we continue our search - } - - registerNewLocsWithTimers(pwns, myName); - - // we've either filled up the cache or run out of elements. Either way we return - // the first element of the cache. If the cache is empty, we return null here. - return cache.poll(); - } - }.run(); - } - } - - public final void remove() { - throw new UnsupportedOperationException(); - } - - public final Iterator iterator() { - return this; - } - } - - // -------------------------------------------------------------------------------- - // - // private / protected low-level accessors / manipulators and utility functions - // - // -------------------------------------------------------------------------------- - - /** - * Useful debugging function that returns the ProcessingLoc who owns loc. ID - * is provided for debugging purposes - * @param loc - * @param id - * @return - */ - protected final ProcessingLoc findOwner(GenomeLoc loc, String id) { - // fast path to check if we already have the existing genome loc in memory for ownership claims - // getProcessingLocs() may be expensive [reading from disk, for example] so we shouldn't call it - // unless necessary - ProcessingLoc x = processingLocs.get(loc); - return x == null ? updateAndGetProcessingLocs(id).get(loc) : x; - } - - /** - * Returns the list of currently owned locations, updating the database as necessary. - * DO NOT MODIFY THIS MAP! As with all parallelizing data structures, the list may be - * out of date immediately after the call returns, or may be updating on the fly. - * @return - */ - protected final Map updateAndGetProcessingLocs(String myName) { - return new WithLock>(myName) { - public Map doBody() { -// readTimer.restart(); - for ( ProcessingLoc p : readNewLocs() ) - processingLocs.put(p.getLocation(), p); -// readTimer.stop(); - nReads++; - return processingLocs; - } - }.run(); - } - - /** - * Wrapper around registerNewLocs that also times the operation - * - * @param plocs - * @param myName - */ - protected final void registerNewLocsWithTimers(Collection plocs, String myName) { -// writeTimer.restart(); - registerNewLocs(plocs); - nWrites++; -// writeTimer.stop(); - } - - private final void printStatusHeader() { - if ( status != null ) status.printf("process.id\thr.time\ttime\tstate%n"); - } - - private final void printStatus(String id, long machineTime, String state) { - // prints a line like processID human-readable-time machine-time state - if ( status != null ) { - status.printf("%s\t%s\t%d\t%s%n", id, STATUS_FORMAT.format(machineTime), machineTime, state); - status.flush(); - } - } - - - /** - * Lock the data structure, preventing other threads / processes from reading and writing to the - * common store - * @param id the name of the process doing the locking - */ - private final void lock(String id) { - //lockWaitTimer.restart(); - boolean hadLock = lock.ownsLock(); - if ( ! hadLock ) { - nLocks++; - //printStatus(id, lockWaitTimer.currentTime(), GOING_FOR_LOCK); - } - lock.lock(); - //lockWaitTimer.stop(); - //if ( ! hadLock ) printStatus(id, lockWaitTimer.currentTime(), HAVE_LOCK); - } - - /** - * Unlock the data structure, allowing other threads / processes to read and write to the common store - * @param id the name of the process doing the unlocking - */ - private final void unlock(String id) { - if ( lock.getHoldCount() == 1 ) printStatus(id, timer.currentTime(), RELEASING_LOCK); - lock.unlock(); - if ( ! lock.ownsLock() ) printStatus(id, timer.currentTime(), RUNNING); - } - - // useful code for getting - public final long getNLocks() { return nLocks; } - public final long getNReads() { return nReads; } - public final long getNWrites() { return nWrites; } -// public final double getTimePerLock() { return lockWaitTimer.getElapsedTime() / Math.max(nLocks, 1); } -// public final double getTimePerRead() { return readTimer.getElapsedTime() / Math.max(nReads,1); } -// public final double getTimePerWrite() { return writeTimer.getElapsedTime() / Math.max(nWrites,1); } - - // -------------------------------------------------------------------------------- - // - // Java-style functional form for with lock do { x }; - // - // -------------------------------------------------------------------------------- - - /** - * Private utility class that executes doBody() method with the lock() acquired and - * handles property unlock()ing the system, even if an error occurs. Allows one to write - * clean code like: - * - * new WithLock(name) { - * public Integer doBody() { doSomething(); return 1; } - * }.run() - * - * @param the return type of the doBody() method - */ - private abstract class WithLock { - private final String myName; - - public WithLock(String myName) { - this.myName = myName; - } - - protected abstract T doBody(); - - public T run() { - boolean locked = false; - try { - lock(myName); - locked = true; - return doBody(); - } finally { - if (locked) unlock(myName); - } - } - } - - // -------------------------------------------------------------------------------- - // - // main function for testing performance - // - // -------------------------------------------------------------------------------- - public static void main(String[] args) { - //BasicConfigurator.configure(); - - final String ref = args[0]; - final File file = new File(args[1]); - final int cycles = Integer.valueOf(args[2]); - - File referenceFile = new File(ref); - try { - final IndexedFastaSequenceFile fasta = new IndexedFastaSequenceFile(referenceFile); - final String chr1 = fasta.getSequenceDictionary().getSequence(1).getSequenceName(); - final GenomeLocParser genomeLocParser = new GenomeLocParser(fasta); - - final class MyTest { - String name; - GenomeLocProcessingTracker tracker; - - MyTest(String name, GenomeLocProcessingTracker tracker) { - this.name = name; - this.tracker = tracker; - } - - public void execute(int cycles) { - SimpleTimer delta = new SimpleTimer("delta"); - SimpleTimer timer = new SimpleTimer("none"); - - if ( file.exists() ) file.delete(); - timer.start(); - delta.start(); - for ( int i = 1; i < cycles; i++ ) { - tracker.claimOwnership(genomeLocParser.createGenomeLoc(chr1, i, i+1), "ABCDEFGHIJKL"); - if ( i % 1000 == 0 ) { - System.out.printf("%s\t%d\t%d\t%.4f\t%.4f%n", name, i, timer.currentTime(), timer.getElapsedTime(), delta.getElapsedTime() ); - delta.restart(); - } - } - } - } - - System.out.printf("name\tcycle\tcurrent.time\telapsed.time\tdelta%n"); - new MyTest("in-memory", new SharedMemoryGenomeLocProcessingTracker(new ClosableReentrantLock())).execute(cycles); - new MyTest("nio", new FileBackedGenomeLocProcessingTracker(file, genomeLocParser, new ClosableReentrantLock(), null)).execute(cycles); - new MyTest("nio-file-lock", new FileBackedGenomeLocProcessingTracker(file, genomeLocParser, new SharedFileThreadSafeLock(file,1), null)).execute(cycles); - } - catch(FileNotFoundException ex) { - throw new UserException.CouldNotReadInputFile(referenceFile,ex); - } - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java deleted file mode 100644 index 9807b6efa..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/NoOpGenomeLocProcessingTracker.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import java.util.Collection; -import java.util.Collections; -import java.util.List; - -/** - * Base class, and null tracker. Always says that a GenomeLoc is ready for processing. It is - * critical that this class already return that a loc is owned, no matter if it's been seen before, - * etc. ReadShards can differ in their contents but have the same "unmapped" genome loc - */ -public class NoOpGenomeLocProcessingTracker extends GenomeLocProcessingTracker { - public NoOpGenomeLocProcessingTracker() { - super(new ClosableReentrantLock(), null); - } - - @Override - protected void registerNewLocs(Collection loc) { - ; - } - - @Override - protected List readNewLocs() { - return Collections.emptyList(); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java deleted file mode 100644 index 0957ac1ae..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/ProcessingLoc.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.HasGenomeLocation; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; - -/** - * Created by IntelliJ IDEA. - * User: depristo - * Date: 1/19/11 - * Time: 8:06 AM - * - * Information about processing locations and their owners. Contains two basic data, associated - * together. The first is a genome loc, and the second is the name of the owner, as a string. - * - * chr1:1-10 Mark - * chr2:11-20 DePristo - * - * would be two ProcessingLocs that first indicate that the first 10 bp of chr1 are owned by Mark, - * and the second is owned by DePristo. - */ -public class ProcessingLoc implements HasGenomeLocation { - private final GenomeLoc loc; - private final String owner; - - /** - * Create a loc that's already owned - * @param loc - * @param owner - */ - public ProcessingLoc(GenomeLoc loc, String owner) { - if ( loc == null || owner == null ) { - throw new ReviewedStingException("BUG: invalid ProcessingLoc detected: " + loc + " owner " + owner); - } - - this.loc = loc; - this.owner = owner.intern(); // reduce memory consumption by interning the string - } - - public GenomeLoc getLocation() { - return loc; - } - - public String getOwner() { - return owner; - } - - /** - * Returns true iff the owner of this processing loc is name. Can be used to determine - * the owner of this processing location. - * - * @param name - * @return - */ - public boolean isOwnedBy(String name) { - return getOwner().equals(name); - } - - public String toString() { return String.format("ProcessingLoc(%s,%s)", loc, owner); } - - public boolean equals(Object other) { - if (other instanceof ProcessingLoc ) - return this.loc.equals(((ProcessingLoc)other).loc) && this.owner.equals(((ProcessingLoc)other).owner); - else - return false; - } - - public int compareTo(ProcessingLoc other) { - return this.getLocation().compareTo(other.getLocation()); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java deleted file mode 100644 index bda62b890..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileLock.java +++ /dev/null @@ -1,171 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import org.apache.log4j.Logger; -import org.apache.lucene.store.*; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.exceptions.UserException; - -import java.io.File; -import java.io.IOException; - -/** - * User: depristo - * Date: 1/19/11 - * Time: 8:24 AM - * - * A reentrant lock for a shared file common file in the file system. Relies on a a Lucene SimpleFSLock - * to manage on disk file locking. - */ -public class SharedFileLock extends ClosableReentrantLock { // todo -- kinda gross inheritance. The super lock is never used - private static Logger logger = Logger.getLogger(SharedFileLock.class); - - private static final String VERIFY_HOST = System.getProperty("verify.host", "gsa1"); - private static final boolean VERIFY = false; - private static final int VERIFY_PORT = 5050; - - // 5 minutes => 360 seconds of trying -> failure - protected static final int DEFAULT_N_TRIES = 1000; - protected static final long DEFAULT_MILLISECONDS_PER_TRY = 360; - - /** The file we are locking */ - private final File file; - - private final LockFactory lockFactory; - private Lock fileLock = null; - - /** - * A counter that indicates the number of 'locks' on this file. - * If locks == 2, then two unlocks are required - * before any resources are freed. - */ - int fileLockReentrantCounter = 0; - - // type of locking - private final int nRetries; - private final long milliSecPerTry; - - /** - * Create a SharedFileThreadSafeLock object locking the file - * @param file - */ - public SharedFileLock(File file, int nRetries, long milliSecPerTry, int ID) { - super(); - this.file = file; - this.nRetries = nRetries; - this.milliSecPerTry = milliSecPerTry; - - File lockDir = new File(file.getParent() == null ? "./" : file.getParent()); - try { - LockFactory factory = new SimpleFSLockFactory(lockDir); - if ( VERIFY ) { // don't forget to start up the VerifyLockServer - this.lockFactory = new VerifyingLockFactory((byte)ID, factory, VERIFY_HOST, VERIFY_PORT); - } else { - this.lockFactory = factory; - } - } catch (IOException e) { - throw new UserException.CouldNotCreateOutputFile(lockDir, "Could not create coordination file locking directory " + lockDir, e); - } - } - - public SharedFileLock(File file, int ID) { - this(file, DEFAULT_N_TRIES, DEFAULT_MILLISECONDS_PER_TRY, ID); - } - - @Override - public void close() { - if ( ownsLock() ) throw new ReviewedStingException("closing SharedFileLock while still owned: ownership count " + fileLockReentrantCounter); - } - - @Override - public int getHoldCount() { - return fileLockReentrantCounter; - } - - @Override - public boolean ownsLock() { - return fileLockReentrantCounter > 0; - } - - // ------------------------------------------------------------------------------------------ - // - // workhorse routines -- acquiring file locks - // - // ------------------------------------------------------------------------------------------ - - private boolean obtainFileLock() throws IOException { - // annoying bug work around for verifylockserver - if ( VERIFY ) - try { - return fileLock.obtain(1); - } catch ( LockObtainFailedException e ) { - return false; - } - else - return fileLock.obtain(); - } - - /** - * Two stage [threading then file] locking mechanism. Reenterant in that multiple lock calls will be - * unwound appropriately. Uses file channel lock *after* thread locking. - */ - @Override - public void lock() { - if ( SharedFileThreadSafeLock.DEBUG ) logger.warn(" lock() " + Thread.currentThread().getName() + ", fileLockReentrantCounter = " + fileLockReentrantCounter); - if ( fileLockReentrantCounter++ == 0 ) { - // Precondition -- lock is always null while we don't have a lock - if ( fileLock != null ) - throw new ReviewedStingException("BUG: lock() function called when a lock already is owned!"); - - int i = 1; - fileLock = lockFactory.makeLock(file.getName() + ".lock"); - try { - boolean obtained = obtainFileLock(); // todo -- maybe use intrinsic lock features - for ( ; ! obtained && i < nRetries; i++ ) { - try { - //logger.warn("tryLock failed on try " + i + ", waiting " + milliSecPerTry + " millseconds for retry"); - Thread.sleep(milliSecPerTry); - } catch ( InterruptedException e ) { - throw new UserException("SharedFileThreadSafeLock interrupted during wait for file lock", e); - } - obtained = obtainFileLock(); // gross workaround for error in verify server - } - - if ( i > 1 ) logger.warn("tryLock required " + i + " tries before completing, waited " + i * milliSecPerTry + " millseconds"); - - if ( ! obtained ) { - fileLock = null; - // filelock == null -> we never managed to acquire the lock! - throw new UserException("SharedFileThreadSafeLock failed to obtain the lock after " + nRetries + " attempts"); - } - - if ( SharedFileThreadSafeLock.DEBUG ) logger.warn(" lock() " + Thread.currentThread().getName() + ", obtained = " + obtained + ", tries = " + i); - } catch (IOException e) { - fileLock = null; - throw new ReviewedStingException("Coordination file could not be created because a lock could not be obtained.", e); - } - } - } - - @Override - public void unlock() { - // update for reentrant unlocking - if ( fileLock == null ) throw new ReviewedStingException("BUG: file lock is null -- file lock was not obtained"); - if ( fileLockReentrantCounter <= 0 ) throw new ReviewedStingException("BUG: file lock counter < 0"); - - // this unlock counts as 1 unlock. If this is our last unlock, actually do something - if ( SharedFileThreadSafeLock.DEBUG ) logger.warn(" unlock() " + Thread.currentThread().getName() + ", count = " + fileLockReentrantCounter); - if ( --fileLockReentrantCounter == 0 ) { - try { - if ( ! fileLock.isLocked() ) throw new ReviewedStingException("BUG: call to unlock() when we don't have a valid lock!"); - fileLock.release(); - if ( SharedFileThreadSafeLock.DEBUG ) logger.warn(" unlock() " + Thread.currentThread().getName() + ", actually releasing"); - } catch ( IOException e ) { - throw new ReviewedStingException("Could not free file lock on file " + file, e); - } finally { // make sure we null out the filelock, regardless of our state - fileLock = null; - } - } else { - if ( SharedFileThreadSafeLock.DEBUG ) logger.warn(" unlock() " + Thread.currentThread().getName() + ", skipping, count = " + fileLockReentrantCounter); - } - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java deleted file mode 100644 index 49fc8208a..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedFileThreadSafeLock.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import org.apache.log4j.Logger; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; - -import java.io.File; - -/** - * User: depristo - * Date: 1/19/11 - * Time: 8:24 AM - * - * A reentrant lock that supports multi-threaded locking as well as a shared file lock on a common - * file in the file system. It itself a shared memory reenterant lock to managed thread safety and - * contains a SharedFileLock to handle the file integrity. - */ -public class SharedFileThreadSafeLock extends ClosableReentrantLock { - private static Logger logger = Logger.getLogger(SharedFileThreadSafeLock.class); - protected static final boolean DEBUG = false; - - private final SharedFileLock fileLock; - - /** - * Create a SharedFileThreadSafeLock object locking the file - * @param file - */ - public SharedFileThreadSafeLock(File file, int nRetries, long milliSecPerTry, int ID) { - super(); - this.fileLock = new SharedFileLock(file, nRetries, milliSecPerTry, ID); - } - - public SharedFileThreadSafeLock(File file, int ID) { - this(file, SharedFileLock.DEFAULT_N_TRIES, SharedFileLock.DEFAULT_MILLISECONDS_PER_TRY, ID); - } - - @Override - public void close() { - super.close(); - fileLock.close(); - } - - @Override - public int getHoldCount() { - if ( super.getHoldCount() != fileLock.getHoldCount() ) - throw new ReviewedStingException("BUG: unequal hold counts. threadlock = " + super.getHoldCount() + ", filelock = " + fileLock.getHoldCount()); - return super.getHoldCount(); - } - - @Override - public boolean ownsLock() { - return super.isHeldByCurrentThread() && fileLock.ownsLock(); - } - - /** - * Two stage [threading then file] locking mechanism. Reenterant in that multiple lock calls will be - * unwound appropriately. Uses file channel lock *after* thread locking. - */ - @Override - public void lock() { - if ( DEBUG ) logger.warn("Attempting SharedFileThreadSafe lock: " + Thread.currentThread().getName()); - if ( DEBUG ) logger.warn(" going for thread lock: " + Thread.currentThread().getName()); - super.lock(); - if ( DEBUG ) logger.warn(" going for file lock: " + Thread.currentThread().getName()); - fileLock.lock(); // todo -- should this be in a try? - } - - @Override - public void unlock() { - if ( DEBUG ) logger.warn(" releasing filelock: " + Thread.currentThread().getName()); - fileLock.unlock(); - if ( DEBUG ) logger.warn(" releasing threadlock: " + Thread.currentThread().getName()); - super.unlock(); - if ( DEBUG ) logger.warn(" unlock() complete: " + Thread.currentThread().getName()); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java deleted file mode 100644 index 0e62d0f63..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/SharedMemoryGenomeLocProcessingTracker.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.broadinstitute.sting.utils.distributedutils; - -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * Thread-safe shared memory only implementation. Uses a simple list to manage the newly - * added processing locations. - */ -public class SharedMemoryGenomeLocProcessingTracker extends GenomeLocProcessingTracker { - private List newPLocs = new ArrayList(); - - protected SharedMemoryGenomeLocProcessingTracker(ClosableReentrantLock lock) { - super(lock, null); - } - - protected SharedMemoryGenomeLocProcessingTracker(ClosableReentrantLock lock, PrintStream status) { - super(lock, status); - } - - @Override - protected void registerNewLocs(Collection plocs) { - newPLocs.addAll(plocs); - } - - @Override - protected List readNewLocs() { - List r = newPLocs; - newPLocs = new ArrayList(); - return r; - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java b/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java deleted file mode 100644 index 033120dc5..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/distributedutils/package-info.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -/** - * Utilities for prototype distributed GATK. No longer in use in the codebase - */ -package org.broadinstitute.sting.utils.distributedutils; diff --git a/public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java deleted file mode 100644 index 139937e29..000000000 --- a/public/java/test/org/broadinstitute/sting/utils/distributedutils/GenomeLocProcessingTrackerUnitTest.java +++ /dev/null @@ -1,402 +0,0 @@ -// our package -package org.broadinstitute.sting.utils.distributedutils; - - -// the imports for unit testing. - - -import net.sf.picard.reference.IndexedFastaSequenceFile; -import org.broadinstitute.sting.BaseTest; -import org.broadinstitute.sting.utils.GenomeLoc; -import org.broadinstitute.sting.utils.GenomeLocParser; -import org.broadinstitute.sting.utils.distributedutils.*; -import org.broadinstitute.sting.utils.exceptions.UserException; -import org.testng.Assert; -import org.testng.annotations.*; - -import java.io.File; -import java.io.FileNotFoundException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.*; - -/** - * Basic unit test for GenomeLoc - */ -public class GenomeLocProcessingTrackerUnitTest extends BaseTest { - IndexedFastaSequenceFile fasta = null; - GenomeLocParser genomeLocParser = null; - String chr1 = null; - private final static String FILE_ROOT = "public/testdata/GLPTFile"; - - @BeforeTest - public void before() { - File referenceFile = new File(hg18Reference); - try { - fasta = new IndexedFastaSequenceFile(referenceFile); - chr1 = fasta.getSequenceDictionary().getSequence(1).getSequenceName(); - genomeLocParser = new GenomeLocParser(fasta); - - } - catch(FileNotFoundException ex) { - throw new UserException.CouldNotReadInputFile(referenceFile,ex); - } - } - - @BeforeMethod - public void beforeMethod(Object[] data) { - if ( data.length > 0 ) - ((TestTarget)data[0]).init(); - } - - @AfterMethod - public void afterMethod(Object[] data) { - if ( data.length > 0 ) { - ((TestTarget)data[0]).getTracker().close(); - ((TestTarget)data[0]).cleanup(); - } - } - - abstract private class TestTarget { - String name; - int nShards; - int shardSize; - File file; - - public void init() { cleanup(); } - - public void cleanup() { - if ( file != null && file.exists() ) - file.delete(); - } - - public boolean isThreadSafe() { return true; } - - protected TestTarget(String name, int nShards, int shardSize, File file) { - this.name = name; - this.nShards = nShards; - this.shardSize = shardSize; - this.file = file; - } - - public abstract GenomeLocProcessingTracker getTracker(); - - public List getShards() { - List shards = new ArrayList(); - for ( int i = 0; i < nShards; i++ ) { - int start = shardSize * i; - int stop = start + shardSize; - shards.add(genomeLocParser.createGenomeLoc(chr1, start, stop)); - } - return shards; - } - - public String toString() { - return String.format("TestTarget %s: nShards=%d shardSize=%d", name, nShards, shardSize); - } - } - - @DataProvider(name = "threadData") - public Object[][] createThreadData() { - // gotta keep the tests small... - return createData(Arrays.asList(10, 100), Arrays.asList(10)); - //return createData(Arrays.asList(10, 100, 1000, 10000), Arrays.asList(10)); - } - - public Object[][] createData(List nShards, List shardSizes) { - List params = new ArrayList(); - - int counter = 0; - String name = null; - for ( int nShard : nShards ) { - for ( int shardSize : shardSizes ) { - // shared mem -- canonical implementation - params.add(new TestTarget("ThreadSafeSharedMemory", nShard, shardSize, null) { - GenomeLocProcessingTracker tracker = new SharedMemoryGenomeLocProcessingTracker(new ClosableReentrantLock()); - public GenomeLocProcessingTracker getTracker() { return tracker; } - }); - - final File file1 = new File(String.format("%s_ThreadSafeFileBacked_%d_%d", FILE_ROOT, counter++, nShard, shardSize)); - params.add(new TestTarget("ThreadSafeFileBacked", nShard, shardSize, file1) { - GenomeLocProcessingTracker tracker = new FileBackedGenomeLocProcessingTracker(file1, genomeLocParser, new ClosableReentrantLock(), null); - public GenomeLocProcessingTracker getTracker() { return tracker; } - }); - - name = "FileBackedSharedFileThreadSafe"; - final File file2 = new File(String.format("%s_%s_%d_%d", FILE_ROOT, name, counter++, nShard, shardSize)); - params.add(new TestTarget(name, nShard, shardSize, file2) { - GenomeLocProcessingTracker tracker = new FileBackedGenomeLocProcessingTracker(file2, genomeLocParser, new SharedFileThreadSafeLock(file2, -1), null); - public GenomeLocProcessingTracker getTracker() { return tracker; } - }); - - name = "FileBackedSharedFile"; - final File file3 = new File(String.format("%s_%s_%d_%d", FILE_ROOT, name, counter++, nShard, shardSize)); - params.add(new TestTarget(name, nShard, shardSize, file3) { - GenomeLocProcessingTracker tracker = new FileBackedGenomeLocProcessingTracker(file3, genomeLocParser, new SharedFileLock(file3, -1), null); - public GenomeLocProcessingTracker getTracker() { return tracker; } - public boolean isThreadSafe() { return false; } - }); - } - } - - List params2 = new ArrayList(); - for ( TestTarget x : params ) params2.add(new Object[]{x}); - return params2.toArray(new Object[][]{}); - } - - @DataProvider(name = "simpleData") - public Object[][] createSimpleData() { - return createData(Arrays.asList(1000), Arrays.asList(100)); - } - - private static final String NAME_ONE = "name1"; - private static final String NAME_TWO = "name2"; - - @Test(enabled = true) - public void testNoop() { - GenomeLocProcessingTracker tracker = new NoOpGenomeLocProcessingTracker(); - for ( int start = 1; start < 100; start++ ) { - for ( int n = 0; n < 2; n++ ) { - GenomeLoc loc = genomeLocParser.createGenomeLoc(chr1, start, start +1); - ProcessingLoc ploc = tracker.claimOwnership(loc, NAME_ONE); - Assert.assertTrue(ploc.isOwnedBy(NAME_ONE)); - Assert.assertEquals(tracker.updateAndGetProcessingLocs(NAME_ONE).size(), 0); - } - } - } - - @Test(dataProvider = "simpleData", enabled = true) - public void testSingleProcessTracker(TestTarget test) { - GenomeLocProcessingTracker tracker = test.getTracker(); - List shards = test.getShards(); - logger.warn("testSingleProcessTracker " + test); - - int counter = 0; - for ( GenomeLoc shard : shards ) { - counter++; - - Assert.assertNull(tracker.findOwner(shard, NAME_ONE)); - Assert.assertFalse(tracker.locIsOwned(shard, NAME_ONE)); - - ProcessingLoc proc = tracker.claimOwnership(shard,NAME_ONE); - Assert.assertNotNull(proc); - Assert.assertNotNull(proc.getLocation()); - Assert.assertNotNull(proc.getOwner()); - Assert.assertEquals(proc.getLocation(), shard); - Assert.assertEquals(proc.getOwner(), NAME_ONE); - Assert.assertEquals(tracker.findOwner(shard, NAME_ONE), proc); - Assert.assertTrue(tracker.locIsOwned(shard, NAME_ONE)); - Assert.assertNotNull(tracker.updateAndGetProcessingLocs(NAME_ONE)); - Assert.assertEquals(tracker.updateAndGetProcessingLocs(NAME_ONE).size(), counter); - - ProcessingLoc badClaimAttempt = tracker.claimOwnership(shard,NAME_TWO); - Assert.assertFalse(badClaimAttempt.getOwner().equals(NAME_TWO)); - Assert.assertEquals(badClaimAttempt.getOwner(), NAME_ONE); - } - } - - @Test(dataProvider = "simpleData", enabled = true) - public void testIterator(TestTarget test) { - GenomeLocProcessingTracker tracker = test.getTracker(); - List shards = test.getShards(); - logger.warn("testIterator " + test); - - List markedShards = new ArrayList(); - List toFind = new ArrayList(); - - for ( int i = 0; i < shards.size(); i++ ) { - if ( ! (i % 10 == 0) ) { - markedShards.add(shards.get(i)); - tracker.claimOwnership(shards.get(i), NAME_TWO); - } else { - toFind.add(shards.get(i)); - } - } - - int nFound = 0; - Iterator it = shards.iterator(); - while ( it.hasNext() ) { - GenomeLoc shard = tracker.claimOwnershipOfNextAvailable(it, NAME_ONE); - - if ( shard == null ) { // everything to get is done - Assert.assertEquals(nFound, toFind.size(), "Didn't find all of the available shards"); - } else { - nFound++; - ProcessingLoc proc = tracker.findOwner(shard, NAME_ONE); - - Assert.assertTrue(proc.isOwnedBy(NAME_ONE)); - Assert.assertTrue(! markedShards.contains(shard), "Ran process was already marked!"); - Assert.assertTrue(toFind.contains(shard), "Claimed shard wasn't one of the unmarked!"); - } - } - } - - @Test(dataProvider = "simpleData", enabled = true) - public void testMarkedProcesses(TestTarget test) { - GenomeLocProcessingTracker tracker = test.getTracker(); - List shards = test.getShards(); - logger.warn("testMarkedProcesses " + test); - - List markedShards = new ArrayList(); - - for ( int i = 0; i < shards.size(); i++ ) { - if ( i % 2 == 0 ) { - markedShards.add(shards.get(i)); - tracker.claimOwnership(shards.get(i), NAME_TWO); - } - } - - for ( GenomeLoc shard : shards ) { - ProcessingLoc proc = tracker.claimOwnership(shard,NAME_ONE); - - Assert.assertTrue(proc.isOwnedBy(NAME_ONE) || proc.isOwnedBy(NAME_TWO)); - - if ( proc.isOwnedBy(NAME_ONE) ) - Assert.assertTrue(! markedShards.contains(shard), "Ran process was already marked!"); - else - Assert.assertTrue(markedShards.contains(shard), "Unran process wasn't marked"); - - if ( ! markedShards.contains(shard) ) { - Assert.assertEquals(tracker.findOwner(shard, NAME_ONE), proc); - } - } - } - - public class TestThread implements Callable { - public TestTarget test; - public String name; - public List ran, toRun; - boolean useIterator; - - public TestThread(TestTarget test, int count, List toRun, boolean useIterator) { - this.test = test; - this.toRun = toRun; - this.name = "thread" + count; - this.ran = new ArrayList(); - this.useIterator = useIterator; - } - - public Integer call() { - //logger.warn(String.format("Call() Thread %s", name)); - if ( useIterator ) { - for ( GenomeLoc shard : test.getTracker().onlyOwned(toRun.iterator(), name) ) { - if ( shard != null ) { // ignore the unclaimable end of the stream - ran.add(shard); - // do some work here - for ( int sum =0, i = 0; i < 100000; i++) sum += i; - } - } - - } else { - for ( GenomeLoc shard : toRun ) { - //System.out.printf("Claiming ownership in %s on %s%n", name, shard); - ProcessingLoc proc = test.getTracker().claimOwnership(shard,name); - //System.out.printf(" => ownership of %s is %s (I own? %b)%n", shard, proc.getOwner(), proc.isOwnedBy(name)); - if ( proc.isOwnedBy(name) ) { - ran.add(proc.getLocation()); - // do some work here - for ( int sum =0, i = 0; i < 100000; i++) sum += i; - } - //logger.warn(String.format("Thread %s on %s -> owned by %s", name, shard, proc.getOwner())); - } - } - - return 1; - } - } - - private static TestThread findOwner(String name, List threads) { - for ( TestThread thread : threads ) { - if ( thread.name.equals(name) ) - return thread; - } - return null; - } - - private static final void assertAllThreadsFinished(List> futures) { - try { - for ( Future f : futures ) { - Assert.assertTrue(f.isDone(), "Thread never finished running"); - Assert.assertTrue(f.get() != null, "Finished successfully"); - } - } catch (InterruptedException e) { - Assert.fail("Thread failed to run to completion", e); - } catch (ExecutionException e) { - Assert.fail("Thread generated an exception", e); - } - } - - private static final List subList(List l, int i) { - List r = new ArrayList(); - for ( int j = 0; j < l.size(); j++ ) { - if ( j % i == 0 ) - r.add(l.get(j)); - } - - return r; - } - - @Test(dataProvider = "threadData", enabled = true) - public void testThreadedProcessesLowLevelFunctions(TestTarget test) { - testThreading(test, false); - } - - @Test(dataProvider = "threadData", enabled = true) - public void testThreadedProcessesIterator(TestTarget test) { - testThreading(test, true); - } - - private void testThreading(TestTarget test, boolean useIterator) { - if ( ! test.isThreadSafe() ) - // skip tests that aren't thread safe - return; - - // start up 3 threads - logger.warn("ThreadedTesting " + test + " using iterator " + useIterator); - List threads = new ArrayList(); - for ( int i = 0; i < 4; i++) { - List toRun = subList(test.getShards(), i+1); - TestThread thread = new TestThread(test, i, toRun, useIterator); - threads.add(thread); - } - ExecutorService exec = java.util.concurrent.Executors.newFixedThreadPool(threads.size()); - - try { - List> results = exec.invokeAll(threads, 300, TimeUnit.SECONDS); - GenomeLocProcessingTracker tracker = test.getTracker(); - List shards = test.getShards(); - - for ( TestThread thread : threads ) - logger.warn(String.format("TestThread %s ran %d jobs of %d to run", thread.name, thread.ran.size(), thread.toRun.size())); - - assertAllThreadsFinished(results); - - // we ran everything - Assert.assertEquals(tracker.updateAndGetProcessingLocs(NAME_ONE).size(), shards.size(), "Not all shards were run"); - - for ( GenomeLoc shard : shards ) { - Assert.assertTrue(tracker.locIsOwned(shard, NAME_ONE), "Unowned shard"); - - ProcessingLoc proc = tracker.findOwner(shard, NAME_ONE); - Assert.assertNotNull(proc, "Proc was null"); - - Assert.assertNotNull(proc.getOwner(), "Owner was null"); - Assert.assertEquals(proc.getLocation(), shard, "Shard loc doesn't make ProcessingLoc"); - - TestThread owner = findOwner(proc.getOwner(), threads); - Assert.assertNotNull(owner, "Couldn't find owner"); - - Assert.assertTrue(owner.ran.contains(shard), "Owner doesn't contain ran shard"); - - for ( TestThread thread : threads ) - if ( ! proc.isOwnedBy(thread.name) && thread.ran.contains(shard) ) - Assert.fail("Shard appears in another run list: proc=" + proc + " shard=" + shard + " also in jobs of " + thread.name + " obj=" + thread.ran.get(thread.ran.indexOf(shard))); - - } - } catch (InterruptedException e) { - Assert.fail("Thread failure", e); - } - } -} \ No newline at end of file From 421b70ca4fffdb98c6f59c7d75e88cba54cfed6a Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 09:42:44 -0400 Subject: [PATCH 10/41] Removed previous, and largely unused, help system extensions. This involved deleting the utils/help/*Taglet.java classes, which parsed out these fields unnecessarily This also involved removing the few uses of these from the codebase. For these uses, though, almost all were an identical copy of the first line of the docs, which is the default javadoc behavior anyway. --- .../sting/gatk/WalkerManager.java | 27 ++---- .../recalibration/CountCovariatesWalker.java | 3 +- .../TableRecalibrationWalker.java | 5 +- .../ApplyRecalibration.java | 2 - .../VariantRecalibrator.java | 2 - .../sting/utils/help/DescriptionTaglet.java | 59 ------------ .../sting/utils/help/DisplayNameTaglet.java | 49 ---------- .../sting/utils/help/HelpTaglet.java | 91 ------------------- .../help/ResourceBundleExtractorDoclet.java | 27 ++---- .../sting/utils/help/SummaryTaglet.java | 58 ------------ 10 files changed, 21 insertions(+), 302 deletions(-) delete mode 100644 public/java/src/org/broadinstitute/sting/utils/help/DescriptionTaglet.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/help/DisplayNameTaglet.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/help/HelpTaglet.java delete mode 100644 public/java/src/org/broadinstitute/sting/utils/help/SummaryTaglet.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/WalkerManager.java b/public/java/src/org/broadinstitute/sting/gatk/WalkerManager.java index cf190835e..6aeb42faa 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/WalkerManager.java +++ b/public/java/src/org/broadinstitute/sting/gatk/WalkerManager.java @@ -33,9 +33,7 @@ import org.broadinstitute.sting.gatk.walkers.*; import org.broadinstitute.sting.utils.baq.BAQ; import org.broadinstitute.sting.utils.classloader.PluginManager; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import org.broadinstitute.sting.utils.help.DescriptionTaglet; -import org.broadinstitute.sting.utils.help.DisplayNameTaglet; -import org.broadinstitute.sting.utils.help.SummaryTaglet; +import org.broadinstitute.sting.utils.help.ResourceBundleExtractorDoclet; import org.broadinstitute.sting.utils.text.TextFormattingUtils; import java.util.*; @@ -82,19 +80,10 @@ public class WalkerManager extends PluginManager { * @return A suitable display name for the package. */ public String getPackageDisplayName(String packageName) { - // Try to find an override for the display name of this package. - String displayNameKey = String.format("%s.%s",packageName,DisplayNameTaglet.NAME); - String displayName; - if(helpText.containsKey(displayNameKey)) { - displayName = helpText.getString(displayNameKey); - } - else { - // If no override exists... - // ...try to compute the override from the text of the package name, while accounting for - // unpackaged walkers. - displayName = packageName.substring(packageName.lastIndexOf('.')+1); - if(displayName.trim().equals("")) displayName = ""; - } + // ...try to compute the override from the text of the package name, while accounting for + // unpackaged walkers. + String displayName = packageName.substring(packageName.lastIndexOf('.')+1); + if (displayName.trim().equals("")) displayName = ""; return displayName; } @@ -104,7 +93,7 @@ public class WalkerManager extends PluginManager { * @return Package help text, or "" if none exists. */ public String getPackageSummaryText(String packageName) { - String key = String.format("%s.%s",packageName,SummaryTaglet.NAME); + String key = String.format("%s.%s",packageName, ResourceBundleExtractorDoclet.SUMMARY_TAGLET_NAME); if(!helpText.containsKey(key)) return ""; return helpText.getString(key); @@ -116,7 +105,7 @@ public class WalkerManager extends PluginManager { * @return Walker summary description, or "" if none exists. */ public String getWalkerSummaryText(Class walkerType) { - String walkerSummary = String.format("%s.%s",walkerType.getName(), SummaryTaglet.NAME); + String walkerSummary = String.format("%s.%s",walkerType.getName(), ResourceBundleExtractorDoclet.SUMMARY_TAGLET_NAME); if(!helpText.containsKey(walkerSummary)) return ""; return helpText.getString(walkerSummary); @@ -137,7 +126,7 @@ public class WalkerManager extends PluginManager { * @return Walker full description, or "" if none exists. */ public String getWalkerDescriptionText(Class walkerType) { - String walkerDescription = String.format("%s.%s",walkerType.getName(), DescriptionTaglet.NAME); + String walkerDescription = String.format("%s.%s",walkerType.getName(), ResourceBundleExtractorDoclet.DESCRIPTION_TAGLET_NAME); if(!helpText.containsKey(walkerDescription)) return ""; return helpText.getString(walkerDescription); diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java index 8c6539f8d..775cde1f4 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/CountCovariatesWalker.java @@ -58,6 +58,8 @@ import java.util.List; import java.util.Map; /** + * First pass of the recalibration. Generates recalibration table based on various user-specified covariates (such as reported quality score, cycle, and dinucleotide). + * * This walker is designed to work as the first pass in a two-pass processing step. * It does a by-locus traversal operating only at sites that are not in dbSNP. * We assume that all reference mismatches we see are therefore errors and indicative of poor base quality. @@ -72,7 +74,6 @@ import java.util.Map; * * @author rpoplin * @since Nov 3, 2009 - * @help.summary First pass of the recalibration. Generates recalibration table based on various user-specified covariates (such as reported quality score, cycle, and dinucleotide). */ @BAQMode(ApplicationTime = BAQ.ApplicationTime.FORBIDDEN) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java index 0277fda0d..fec7ee4e6 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/recalibration/TableRecalibrationWalker.java @@ -54,8 +54,10 @@ import java.util.ResourceBundle; import java.util.regex.Pattern; /** + * Second pass of the recalibration. Uses the table generated by CountCovariates to update the base quality scores of the input bam file using a sequential table calculation making the base quality scores more accurately reflect the actual quality of the bases as measured by reference mismatch rate. + * * This walker is designed to work as the second pass in a two-pass processing step, doing a by-read traversal. - + * * For each base in each read this walker calculates various user-specified covariates (such as read group, reported quality score, cycle, and dinuc) * Using these values as a key in a large hashmap the walker calculates an empirical base quality score and overwrites the quality score currently in the read. * This walker then outputs a new bam file with these updated (recalibrated) reads. @@ -65,7 +67,6 @@ import java.util.regex.Pattern; * * @author rpoplin * @since Nov 3, 2009 - * @help.summary Second pass of the recalibration. Uses the table generated by CountCovariates to update the base quality scores of the input bam file using a sequential table calculation making the base quality scores more accurately reflect the actual quality of the bases as measured by reference mismatch rate. */ @BAQMode(QualityMode = BAQ.QualityMode.ADD_TAG, ApplicationTime = BAQ.ApplicationTime.ON_OUTPUT) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/ApplyRecalibration.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/ApplyRecalibration.java index 403c67d3e..b195fd35f 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/ApplyRecalibration.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/ApplyRecalibration.java @@ -49,8 +49,6 @@ import java.util.*; * * @author rpoplin * @since Mar 14, 2011 - * - * @help.summary Applies cuts to the input vcf file (by adding filter lines) to achieve the desired novel FDR levels which were specified during VariantRecalibration */ public class ApplyRecalibration extends RodWalker { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java index 8179463eb..76c888640 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/variantrecalibration/VariantRecalibrator.java @@ -53,8 +53,6 @@ import java.util.*; * * User: rpoplin * Date: 3/12/11 - * - * @help.summary Takes variant calls as .vcf files, learns a Gaussian mixture model over the variant annotations and evaluates the variant -- assigning an informative lod score */ public class VariantRecalibrator extends RodWalker, ExpandingArrayList> implements TreeReducible> { diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DescriptionTaglet.java b/public/java/src/org/broadinstitute/sting/utils/help/DescriptionTaglet.java deleted file mode 100644 index 65c332048..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/help/DescriptionTaglet.java +++ /dev/null @@ -1,59 +0,0 @@ -package org.broadinstitute.sting.utils.help; - -import com.sun.tools.doclets.Taglet; - -import java.util.Map; - -/** - * Provide an alternate description for the given help system. - * - * @author mhanna - * @version 0.1 - */ -public class DescriptionTaglet extends HelpTaglet { - /** - * The key tag for this taglet. - */ - public static final String NAME = "help.description"; - - /** - * Return the name of this custom tag. - */ - @Override - public String getName() { - return NAME; - } - - /** - * Will return false since overviews are always named - * by the @WalkerName tag. - * @return false always - */ - @Override - public boolean inOverview() { - return true; - } - - /** - * Will return true to indicate that packages can be given useful - * description. - * @return true always - */ - @Override - public boolean inPackage() { - return true; - } - - /** - * Register this Taglet. - * @param tagletMap the map to register this tag to. - */ - public static void register(Map tagletMap) { - DescriptionTaglet tag = new DescriptionTaglet(); - Taglet t = (Taglet)tagletMap.get(tag.getName()); - if (t != null) { - tagletMap.remove(tag.getName()); - } - tagletMap.put(tag.getName(), tag); - } -} \ No newline at end of file diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DisplayNameTaglet.java b/public/java/src/org/broadinstitute/sting/utils/help/DisplayNameTaglet.java deleted file mode 100644 index 6c6dad736..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/help/DisplayNameTaglet.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.broadinstitute.sting.utils.help; - -import com.sun.tools.doclets.Taglet; - -import java.util.Map; - -/** - * Provide a display name in the help for packages - * - * @author mhanna - * @version 0.1 - */ -public class DisplayNameTaglet extends HelpTaglet { - /** - * The display name for this taglet. - */ - public static final String NAME = "help.display.name"; - - /** - * Return the name of this custom tag. - */ - @Override - public String getName() { - return NAME; - } - - /** - * Will return true to indicate that packages can be given useful - * display text. - * @return true always - */ - @Override - public boolean inPackage() { - return true; - } - - /** - * Register this Taglet. - * @param tagletMap the map to register this tag to. - */ - public static void register(Map tagletMap) { - DisplayNameTaglet tag = new DisplayNameTaglet(); - Taglet t = (Taglet)tagletMap.get(tag.getName()); - if (t != null) { - tagletMap.remove(tag.getName()); - } - tagletMap.put(tag.getName(), tag); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/HelpTaglet.java b/public/java/src/org/broadinstitute/sting/utils/help/HelpTaglet.java deleted file mode 100644 index b350b1a29..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/help/HelpTaglet.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.broadinstitute.sting.utils.help; - -import com.sun.javadoc.Tag; -import com.sun.tools.doclets.Taglet; - -/** - * Basic functionality for the help taglet. - * - * @author mhanna - * @version 0.1 - */ -public abstract class HelpTaglet implements Taglet { - /** - * Return the name of this custom tag. - */ - public abstract String getName(); - - /** - * Will return false since this tag cannot be applied - * to a field. - * @return false since this tag cannot be applied to a field. - */ - public boolean inField() { - return false; - } - - /** - * Will return false since by default, help tags cannot be applied to a constructor. - * @return false since by default, help tags cannot be applied to a constructor. - */ - public boolean inConstructor() { - return false; - } - - /** - * Will return false since by default, help tags cannot be applied to a method. - * @return false since by default, this tag cannot be applied to a method. - */ - public boolean inMethod() { - return false; - } - - /** - * Will return false since by default, help tags cannot be applied to an overview. - * @return false since by default, help tags cannot be applied to an overview. - */ - public boolean inOverview() { - return false; - } - - /** - * Will return false since by default, help tags cannot be applied to a package. - * description. - * @return false since by default, help tags cannot be applied to a package. - */ - public boolean inPackage() { - return false; - } - - /** - * Will return false since help tags are by default not inline. - * @return false since help tags are by default not inline. - */ - public boolean inType() { - return false; - } - - /** - * Will return false since help tags are by default not inline. - * @return false since help tags are by default not inline. - */ - public boolean isInlineTag() { - return false; - } - - /** - * Create a string representation of this tag. Since this tag is only - * used by the help system, don't output any HTML. - */ - public String toString(Tag tag) { - return null; - } - - /** - * Create a string representation of this tag. Since this tag is only - * used by the help system, don't output any HTML. - */ - public String toString(Tag[] tags) { - return null; - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 4f4c411a3..c7b8b045e 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -47,7 +47,9 @@ public class ResourceBundleExtractorDoclet { /** * Taglet for the particular version number. */ - protected static final String VERSION_TAGLET_NAME = "version"; + public static final String VERSION_TAGLET_NAME = "version"; + public static final String SUMMARY_TAGLET_NAME = "help.summary"; + public static final String DESCRIPTION_TAGLET_NAME = "help.description"; /** * Maintains a collection of resources in memory as they're accumulated. @@ -250,39 +252,26 @@ public class ResourceBundleExtractorDoclet { String description = element.commentText(); for(Tag tag: element.tags()) { - if(tag.name().equals("@"+DisplayNameTaglet.NAME)) { - if(name != null) - throw new ReviewedStingException("Only one display name tag can be used per package / walker."); - name = tag.text(); - } - else if(tag.name().equals("@"+VERSION_TAGLET_NAME)) { + if(tag.name().equals("@"+VERSION_TAGLET_NAME)) { if ( absoluteVersion != null ) { version = absoluteVersion; } else { version = String.format("%s%s%s", (versionPrefix != null) ? versionPrefix : "", - tag.text(), - (versionSuffix != null) ? versionSuffix : ""); + tag.text(), + (versionSuffix != null) ? versionSuffix : ""); } } - else if(tag.name().equals("@"+SummaryTaglet.NAME)) - summary = tag.text(); - else if(tag.name().equals("@"+DescriptionTaglet.NAME)) - description = tag.text(); } - // Write out an alternate element name, if exists. - if(name != null) - resourceText.setProperty(String.format("%s.%s",elementName,DisplayNameTaglet.NAME),name); - if(version != null) resourceText.setProperty(String.format("%s.%s",elementName,VERSION_TAGLET_NAME),version); // Write out an alternate element summary, if exists. - resourceText.setProperty(String.format("%s.%s",elementName,SummaryTaglet.NAME),formatText(summary)); + resourceText.setProperty(String.format("%s.%s",elementName,SUMMARY_TAGLET_NAME),formatText(summary)); // Write out an alternate description, if present. - resourceText.setProperty(String.format("%s.%s",elementName,DescriptionTaglet.NAME),formatText(description)); + resourceText.setProperty(String.format("%s.%s",elementName,DESCRIPTION_TAGLET_NAME),formatText(description)); } /** diff --git a/public/java/src/org/broadinstitute/sting/utils/help/SummaryTaglet.java b/public/java/src/org/broadinstitute/sting/utils/help/SummaryTaglet.java deleted file mode 100644 index db8b55940..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/help/SummaryTaglet.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.broadinstitute.sting.utils.help; - -import com.sun.tools.doclets.Taglet; - -import java.util.Map; - -/** - * Provide an alternate brief summary for this walker / package. - * Acts as an alternative to the first sentence employed by default. - * @author mhanna - * @version 0.1 - */ -public class SummaryTaglet extends HelpTaglet { - /** - * The key tag for this taglet. - */ - public static final String NAME = "help.summary"; - - /** - * Return the name of this custom tag. - */ - @Override - public String getName() { - return NAME; - } - - /** - * Will return false since overviews are always named - * by the @WalkerName tag. - * @return false always - */ - @Override - public boolean inOverview() { - return true; - } - - /** - * Will return true to indicate that packages can be given useful summary. - * @return true always - */ - @Override - public boolean inPackage() { - return true; - } - - /** - * Register this Taglet. - * @param tagletMap the map to register this tag to. - */ - public static void register(Map tagletMap) { - SummaryTaglet tag = new SummaryTaglet(); - Taglet t = (Taglet)tagletMap.get(tag.getName()); - if (t != null) { - tagletMap.remove(tag.getName()); - } - tagletMap.put(tag.getName(), tag); - } -} \ No newline at end of file From 9e88d51db9043ba28853d89a869bef6cdc75cee4 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 09:57:03 -0400 Subject: [PATCH 13/41] Removed now unused @version tags from walker docs. --- .../walkers/diffengine/DiffObjectsWalker.java | 1 - .../gatk/walkers/qc/CountPairsWalker.java | 1 - .../help/ResourceBundleExtractorDoclet.java | 29 ++++--------------- 3 files changed, 5 insertions(+), 26 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index 899c3671c..98b1eb13d 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -115,7 +115,6 @@ import java.util.List; * * @author Mark DePristo * @since 7/4/11 - * @version 0.1 */ @Requires(value={}) public class DiffObjectsWalker extends RodWalker { diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountPairsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountPairsWalker.java index df89efe6d..26fa9a258 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountPairsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/qc/CountPairsWalker.java @@ -40,7 +40,6 @@ import java.util.List; * of paired reads. * * @author mhanna - * @version 0.1 */ public class CountPairsWalker extends ReadPairWalker { @Output diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index c7b8b045e..140e06481 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -61,7 +61,7 @@ public class ResourceBundleExtractorDoclet { */ protected final Set undocumentedWalkers = new HashSet(); - protected String buildTimestamp = null, versionPrefix = null, versionSuffix = null, absoluteVersion = null; + protected String buildTimestamp = null, absoluteVersion = null; /** * Extracts the contents of certain types of javadoc and adds them to an XML file. @@ -93,10 +93,6 @@ public class ResourceBundleExtractorDoclet { } if(options[0].equals("-build-timestamp")) buildTimestamp = options[1]; - if(options[0].equals("-version-prefix")) - versionPrefix = options[1]; - if(options[0].equals("-version-suffix")) - versionSuffix = options[1]; if (options[0].equals("-absolute-version")) absoluteVersion = options[1]; } @@ -144,7 +140,7 @@ public class ResourceBundleExtractorDoclet { * @return Number of potential parameters; 0 if not supported. */ public static int optionLength(String option) { - if(option.equals("-build-timestamp") || option.equals("-version-prefix") || option.equals("-version-suffix") || option.equals("-out") || option.equals("-absolute-version") ) { + if(option.equals("-build-timestamp") || option.equals("-out") || option.equals("-absolute-version") ) { return 2; } return 0; @@ -242,30 +238,15 @@ public class ResourceBundleExtractorDoclet { * @param element Doc element to process. */ private void renderHelpText(String elementName, Doc element) { - // Extract overrides from the doc tags. - String name = null; - String version = null; StringBuilder summaryBuilder = new StringBuilder(); for(Tag tag: element.firstSentenceTags()) summaryBuilder.append(tag.text()); String summary = summaryBuilder.toString(); String description = element.commentText(); - for(Tag tag: element.tags()) { - if(tag.name().equals("@"+VERSION_TAGLET_NAME)) { - if ( absoluteVersion != null ) { - version = absoluteVersion; - } - else { - version = String.format("%s%s%s", (versionPrefix != null) ? versionPrefix : "", - tag.text(), - (versionSuffix != null) ? versionSuffix : ""); - } - } - } - - if(version != null) - resourceText.setProperty(String.format("%s.%s",elementName,VERSION_TAGLET_NAME),version); + // this might seem unnecessary, but the GATK command line program uses this tag to determine the version when running + if(absoluteVersion != null) + resourceText.setProperty(String.format("%s.%s",elementName,VERSION_TAGLET_NAME),absoluteVersion); // Write out an alternate element summary, if exists. resourceText.setProperty(String.format("%s.%s",elementName,SUMMARY_TAGLET_NAME),formatText(summary)); From 453954182e8332d4c01c63359ad45bbca16521a9 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 13:18:33 -0400 Subject: [PATCH 15/41] Generalized the documentation system to use a class-specific annotation and processor. Need to generalize and bug fix the system. But at a high level it's working now. --- .../sting/gatk/CommandLineGATK.java | 1 + .../sting/gatk/filters/ReadFilter.java | 1 + .../sting/gatk/walkers/Walker.java | 3 + .../varianteval/stratifications/CpG.java | 12 + .../utils/help/DocumentedGATKFeature.java | 40 +++ .../help/DocumentedGATKFeatureHandler.java | 70 +++++ .../sting/utils/help/GATKDoclet.java | 263 +++++++----------- .../help/WalkerDocumentationHandler.java | 196 +++++++++++++ .../helpTemplates/walker.index.template.html | 33 ++- 9 files changed, 449 insertions(+), 170 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index da2be74bf..3726e8e02 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -34,6 +34,7 @@ import org.broadinstitute.sting.gatk.walkers.Attribution; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.exceptions.UserException; import org.broadinstitute.sting.utils.help.ApplicationDetails; +import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; import org.broadinstitute.sting.utils.text.TextFormattingUtils; import java.util.*; diff --git a/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java b/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java index 227637761..003d9cd42 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java +++ b/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java @@ -2,6 +2,7 @@ package org.broadinstitute.sting.gatk.filters; import net.sf.picard.filter.SamRecordFilter; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; +import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; /** * A SamRecordFilter that also depends on the header. diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java index 384742302..24813f4c6 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java @@ -31,6 +31,8 @@ import org.broadinstitute.sting.gatk.filters.MalformedReadFilter; import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.baq.BAQ; import org.broadinstitute.sting.utils.collections.Pair; +import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; +import org.broadinstitute.sting.utils.help.WalkerDocumentationHandler; import java.util.List; @@ -44,6 +46,7 @@ import java.util.List; @ReadFilters(MalformedReadFilter.class) @PartitionBy(PartitionType.NONE) @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT) +@DocumentedGATKFeature( handler = WalkerDocumentationHandler.class ) public abstract class Walker { final protected static Logger logger = Logger.getLogger(Walker.class); private GenomeAnalysisEngine toolkit; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/CpG.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/CpG.java index 3e8a6ed17..e1f2ae983 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/CpG.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/varianteval/stratifications/CpG.java @@ -8,6 +8,18 @@ import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.util.ArrayList; import java.util.Set; +/** + * CpG is a stratification module for VariantEval that divides the input data by within/not within a CpG site + * + *

+ * It is a three-state stratification: + *

    + *
  • The locus is a CpG site ("CpG") + *
  • The locus is not a CpG site ("non_CpG") + *
  • The locus is either a CpG or not a CpG site ("all") + *
+ * A CpG site is defined as a site where the reference base at a locus is a C and the adjacent reference base in the 3' direction is a G. + */ public class CpG extends VariantStratifier { private ArrayList states; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java new file mode 100644 index 000000000..02a7707e6 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import java.lang.annotation.*; + +/** + * An annotation to identify a class as a GATK capability for documentation + * + * @author depristo + */ +@Documented +@Inherited +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.TYPE) +public @interface DocumentedGATKFeature { + Class handler(); +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java new file mode 100644 index 000000000..e459e1543 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.FieldDoc; +import com.sun.javadoc.RootDoc; +import com.sun.javadoc.Tag; +import freemarker.template.Configuration; +import freemarker.template.DefaultObjectWrapper; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.CommandLineGATK; +import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + +import java.io.*; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + */ +public abstract class DocumentedGATKFeatureHandler { + private GATKDoclet doclet; + + protected RootDoc getRootDoc() { + return this.doclet.rootDoc; + } + + public void setDoclet(GATKDoclet doclet) { + this.doclet = doclet; + } + + public boolean shouldBeProcessed(ClassDoc doc) { return true; } + + public String getDestinationFilename(ClassDoc doc) { + return ResourceBundleExtractorDoclet.getClassName(doc).replace(".", "_") + ".html"; + } + + public abstract String getGroupName(); + public abstract String getTemplateName(ClassDoc doc) throws IOException; + public abstract GATKDoclet.DocumentationData processOne(ClassDoc doc); +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index b6488cff9..e155923a5 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -24,29 +24,57 @@ package org.broadinstitute.sting.utils.help; -import com.sun.javadoc.*; +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.RootDoc; import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; -import org.broadinstitute.sting.commandline.*; -import org.broadinstitute.sting.gatk.CommandLineExecutable; -import org.broadinstitute.sting.gatk.CommandLineGATK; -import org.broadinstitute.sting.gatk.walkers.Walker; -import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.classloader.JVMUtils; +import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import scala.reflect.Print; -import sun.tools.java.ClassNotFound; import java.io.*; -import java.lang.reflect.Field; import java.util.*; /** * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { + final protected static Logger logger = Logger.getLogger(GATKDoclet.class); + + public static class DocumentationData implements Comparable { + String name, summary, filename; + Map forTemplate; + String group; + + public DocumentationData(String name, String summary, Map forTemplate) { + this.name = name; + this.summary = summary; + this.forTemplate = forTemplate; + } + + public void setGroup(String group) { + this.group = group; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + public Map toMap() { + Map data = new HashMap(); + data.put("name", name); + data.put("summary", summary); + data.put("filename", filename); + data.put("group", group); + return data; + } + + public int compareTo(DocumentationData other) { + return this.name.compareTo(other.name); + } + } + RootDoc rootDoc; /** @@ -57,7 +85,6 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { */ public static boolean start(RootDoc rootDoc) throws IOException { GATKDoclet doclet = new GATKDoclet(); - //PrintStream out = doclet.loadData(rootDoc, false); doclet.processDocs(rootDoc, null); return true; } @@ -77,20 +104,20 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Configuration cfg = new Configuration(); // Specify the data source where the template files come from. - // Here I set a file directory for it: cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/")); // Specify how templates will see the data-model. This is an advanced topic... - // but just use this: cfg.setObjectWrapper(new DefaultObjectWrapper()); - List> indexData = new ArrayList>(); + List indexData = new ArrayList(); for ( ClassDoc doc : rootDoc.classes() ) { - if ( ResourceBundleExtractorDoclet.isWalker(doc) ) { // && getClassName(doc).contains("UGCalcLikelihoods")) { - System.out.printf("Walker class %s%n", doc); - indexData.add(processWalkerDocs(cfg, doc)); + DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc); + if ( handler != null && handler.shouldBeProcessed(doc) ) { + DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc); + indexData.add(docData); } } - processWalkerIndex(indexData,cfg); + + processIndex(cfg, indexData); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -98,171 +125,91 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - private void processWalkerIndex(List> indexData, Configuration cfg) throws IOException { + private DocumentedGATKFeatureHandler getHandlerForClassDoc(ClassDoc doc) { + try { + // todo -- what do I need the ? extends Object to pass the compiler? + Class docClass = ResourceBundleExtractorDoclet.getClassForDoc(doc); + if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) { + DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class); + DocumentedGATKFeatureHandler handler = feature.handler().newInstance(); + handler.setDoclet(this); + return handler; + } else { + return null; // not annotated so it shouldn't be documented + } + } catch ( ClassNotFoundException e) { + //logger.warn("Couldn't find class for ClassDoc " + doc); + // we got a classdoc for a class we can't find. Maybe in a library or something + return null; + } catch ( IllegalAccessException e) { + throw new RuntimeException(e); // the constructor is now private -- this is an error + } catch ( InstantiationException e) { + throw new RuntimeException(e); // the constructor is now private -- this is an error + } catch ( UnsatisfiedLinkError e) { + return null; // naughty BWA bindings + } + } + + private void processIndex(Configuration cfg, List indexData) throws IOException { /* Get or create a template */ Template temp = cfg.getTemplate("walker.index.template.html"); /* Merge data-model with template */ Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); try { - Map root = new HashMap(); - root.put("walkers", indexData); - temp.process(root, out); + temp.process(groupIndexData(indexData), out); out.flush(); } catch ( TemplateException e ) { throw new ReviewedStingException("Failed to create GATK documentation", e); } } - private Map processWalkerDocs(Configuration cfg, ClassDoc doc) throws IOException { - // Create the root hash - Map root = buildWalkerDataModel(doc); - - /* Get or create a template */ - Template temp = cfg.getTemplate("walker.template.html"); - - /* Merge data-model with template */ - File outputFile = new File(getClassName(doc).replace(".", "_") + ".html"); - File outputPath = new File("testdoc/" + outputFile); - try { - Writer out = new OutputStreamWriter(new FileOutputStream(outputPath)); - temp.process(root, out); - out.flush(); - } catch ( TemplateException e ) { - throw new ReviewedStingException("Failed to create GATK documentation", e); - } - - // add index data - Map indexData = new HashMap(); - indexData.put("filename", outputFile.toString()); - indexData.put("name", doc.name()); - indexData.put("summary", root.get("summary")); - return indexData; - } - - - private Map buildWalkerDataModel(ClassDoc classdoc) { + private Map groupIndexData(List indexData) { + // + // root -> data -> { summary -> y, filename -> z }, etc + // -> groups -> group1, group2, etc. Map root = new HashMap(); - root.put("name", classdoc.name()); + Collections.sort(indexData); - // Extract overrides from the doc tags. - StringBuilder summaryBuilder = new StringBuilder(); - for(Tag tag: classdoc.firstSentenceTags()) - summaryBuilder.append(tag.text()); - root.put("summary", summaryBuilder.toString()); - root.put("description", classdoc.commentText()); + List> data = new ArrayList>(); + Set groups = new HashSet(); - for(Tag tag: classdoc.tags()) { - root.put(tag.name(), tag.text()); + for ( DocumentationData indexDatum : indexData ) { + data.add(indexDatum.toMap()); + groups.add(indexDatum.group); } - ParsingEngine parsingEngine = createStandardGATKParsingEngine(); -// for (ArgumentDefinition argumentDefinition : parsingEngine.argumentDefinitions ) -// System.out.println(argumentDefinition); + root.put("data", data); + root.put("groups", new ArrayList(groups)); - Map> args = new HashMap>(); - root.put("arguments", args); - args.put("all", new ArrayList()); - args.put("required", new ArrayList()); - args.put("optional", new ArrayList()); - args.put("hidden", new ArrayList()); - args.put("depreciated", new ArrayList()); - try { - for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(getClassForDoc(classdoc)) ) { - ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); - FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); - GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? - String kind = "optional"; - if ( argumentSource.isRequired() ) kind = "required"; - else if ( argumentSource.isHidden() ) kind = "hidden"; - else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(doc.toDataModel()); - args.get("all").add(doc.toDataModel()); - System.out.printf("Processing %s%n", argumentSource); - } - } catch ( ClassNotFoundException e ) { - throw new RuntimeException(e); - } - - System.out.printf("Root is %s%n", root); return root; } -// protected String withDefault(String val, String def) { -// return val == null ? def : val; -// } + private DocumentationData processDocumentationWithHandler(Configuration cfg, + DocumentedGATKFeatureHandler handler, + ClassDoc doc) + throws IOException { + System.out.printf("Processing documentation for class %s%n", doc); - protected ParsingEngine createStandardGATKParsingEngine() { - CommandLineProgram clp = new CommandLineGATK(); + DocumentationData docData = handler.processOne(doc); + docData.setGroup(handler.getGroupName()); + + // Get or create a template + Template temp = cfg.getTemplate(handler.getTemplateName(doc)); + + // Merge data-model with template + String filename = handler.getDestinationFilename(doc); + docData.setFilename(filename); + File outputPath = new File("testdoc/" + filename); try { - CommandLineProgram.start(clp, new String[]{}, true); - return clp.parser; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - private FieldDoc getFieldDoc(ClassDoc classDoc, String name) { - return getFieldDoc(classDoc, name, true); - } - - private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { - System.out.printf("Looking for %s in %s%n", name, classDoc.name()); - for ( FieldDoc fieldDoc : classDoc.fields(false) ) { - System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); - if ( fieldDoc.name().equals(name) ) - return fieldDoc; - - Field field = getFieldForFieldDoc(fieldDoc); - if ( field.isAnnotationPresent(ArgumentCollection.class) ) { - ClassDoc typeDoc = this.rootDoc.classNamed(fieldDoc.type().qualifiedTypeName()); - if ( typeDoc == null ) - throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); - else { - FieldDoc result = getFieldDoc(typeDoc, name, false); - if ( result != null ) - return result; - // else keep searching - } - } + Writer out = new OutputStreamWriter(new FileOutputStream(outputPath)); + temp.process(docData.forTemplate, out); + out.flush(); + } catch ( TemplateException e ) { + throw new ReviewedStingException("Failed to create GATK documentation", e); } - // if we didn't find it here, wander up to the superclass to find the field - if ( classDoc.superclass() != null ) { - return getFieldDoc(classDoc.superclass(), name, false); - } - - if ( primary ) - throw new RuntimeException("No field found for expected field " + name); - else - return null; - } - - protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { - final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; - GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); - - if ( def.fullName != null && def.shortName != null) - doc.addSynonym("-" + def.shortName); - - doc.addTag("required", def.required ? "yes" : "no"); - doc.addTag("type", def.argumentType.getSimpleName()); - if ( def.doc != null ) doc.setSummary(def.doc); - - List attributes = new ArrayList(); - attributes.add(def.ioType.annotationClass.getSimpleName()); - if ( def.required ) attributes.add("required"); - if ( def.isFlag ) attributes.add("flag"); - if ( def.isHidden ) attributes.add("hidden"); - doc.addTag("attributes", Utils.join(",", attributes)); - - // todo -- need depreciated value - - doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); - - doc.setFulltext(fieldDoc.commentText()); - - return doc; + return docData; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java new file mode 100644 index 000000000..f9109db72 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import com.sun.javadoc.ClassDoc; +import com.sun.javadoc.FieldDoc; +import com.sun.javadoc.RootDoc; +import com.sun.javadoc.Tag; +import freemarker.template.Configuration; +import freemarker.template.DefaultObjectWrapper; +import freemarker.template.Template; +import freemarker.template.TemplateException; +import org.broadinstitute.sting.commandline.*; +import org.broadinstitute.sting.gatk.CommandLineGATK; +import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.classloader.JVMUtils; +import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; + +import java.io.*; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + */ +public class WalkerDocumentationHandler extends DocumentedGATKFeatureHandler { + @Override + public boolean shouldBeProcessed(ClassDoc doc) { + try { + Class type = ResourceBundleExtractorDoclet.getClassForDoc(doc); + return JVMUtils.isConcrete(type); + } catch ( ClassNotFoundException e ) { + return false; + } + } + + @Override + public String getGroupName() { return "GATK Walkers"; } + + @Override + public String getTemplateName(ClassDoc doc) throws IOException { + return "walker.template.html"; + } + + @Override + public GATKDoclet.DocumentationData processOne(ClassDoc doc) { + System.out.printf("Walker class %s%n", doc); + Map root = buildWalkerDataModel(doc); // Create the root hash + return new GATKDoclet.DocumentationData(doc.name(), (String)root.get("summary"), root); + } + + + private Map buildWalkerDataModel(ClassDoc classdoc) { + Map root = new HashMap(); + + root.put("name", classdoc.name()); + + // Extract overrides from the doc tags. + StringBuilder summaryBuilder = new StringBuilder(); + for(Tag tag: classdoc.firstSentenceTags()) + summaryBuilder.append(tag.text()); + root.put("summary", summaryBuilder.toString()); + root.put("description", classdoc.commentText()); + + for(Tag tag: classdoc.tags()) { + root.put(tag.name(), tag.text()); + } + + ParsingEngine parsingEngine = createStandardGATKParsingEngine(); + + Map> args = new HashMap>(); + root.put("arguments", args); + args.put("all", new ArrayList()); + args.put("required", new ArrayList()); + args.put("optional", new ArrayList()); + args.put("hidden", new ArrayList()); + args.put("depreciated", new ArrayList()); + try { + for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(ResourceBundleExtractorDoclet.getClassForDoc(classdoc)) ) { + ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); + FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); + GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? + String kind = "optional"; + if ( argumentSource.isRequired() ) kind = "required"; + else if ( argumentSource.isHidden() ) kind = "hidden"; + else if ( argumentSource.isDeprecated() ) kind = "depreciated"; + args.get(kind).add(doc.toDataModel()); + args.get("all").add(doc.toDataModel()); + System.out.printf("Processing %s%n", argumentSource); + } + } catch ( ClassNotFoundException e ) { + throw new RuntimeException(e); + } + + //System.out.printf("Root is %s%n", root); + return root; + } + + protected ParsingEngine createStandardGATKParsingEngine() { + CommandLineProgram clp = new CommandLineGATK(); + try { + CommandLineProgram.start(clp, new String[]{}, true); + return clp.parser; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private FieldDoc getFieldDoc(ClassDoc classDoc, String name) { + return getFieldDoc(classDoc, name, true); + } + + private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { + System.out.printf("Looking for %s in %s%n", name, classDoc.name()); + for ( FieldDoc fieldDoc : classDoc.fields(false) ) { + System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); + if ( fieldDoc.name().equals(name) ) + return fieldDoc; + + Field field = ResourceBundleExtractorDoclet.getFieldForFieldDoc(fieldDoc); + if ( field.isAnnotationPresent(ArgumentCollection.class) ) { + ClassDoc typeDoc = getRootDoc().classNamed(fieldDoc.type().qualifiedTypeName()); + if ( typeDoc == null ) + throw new ReviewedStingException("Tried to get javadocs for ArgumentCollection field " + fieldDoc + " but could't find the class in the RootDoc"); + else { + FieldDoc result = getFieldDoc(typeDoc, name, false); + if ( result != null ) + return result; + // else keep searching + } + } + } + + // if we didn't find it here, wander up to the superclass to find the field + if ( classDoc.superclass() != null ) { + return getFieldDoc(classDoc.superclass(), name, false); + } + + if ( primary ) + throw new RuntimeException("No field found for expected field " + name); + else + return null; + } + + protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { + final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; + GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); + + if ( def.fullName != null && def.shortName != null) + doc.addSynonym("-" + def.shortName); + + doc.addTag("required", def.required ? "yes" : "no"); + doc.addTag("type", def.argumentType.getSimpleName()); + if ( def.doc != null ) doc.setSummary(def.doc); + + List attributes = new ArrayList(); + attributes.add(def.ioType.annotationClass.getSimpleName()); + if ( def.required ) attributes.add("required"); + if ( def.isFlag ) attributes.add("flag"); + if ( def.isHidden ) attributes.add("hidden"); + doc.addTag("attributes", Utils.join(",", attributes)); + + // todo -- need depreciated value + + doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); + + doc.setFulltext(fieldDoc.commentText()); + + return doc; + } +} diff --git a/settings/helpTemplates/walker.index.template.html b/settings/helpTemplates/walker.index.template.html index 5d4cd84d4..83020a6e0 100644 --- a/settings/helpTemplates/walker.index.template.html +++ b/settings/helpTemplates/walker.index.template.html @@ -1,21 +1,30 @@ - - - GATK walker index - - -

GATK walker index

+<#macro emitGroup group> +

${group}

- <#list walkers as walker> - - - - - + <#list data as datum> + <#if datum.group == group> + + + + + +
Name Summary
${walker.name}${walker.summary}
${datum.name}${datum.summary}
+ + + + + GATK documentation index + + +

GATK documentation index

+ <#list groups as group> + <@emitGroup group=group/> + From f0be7348be7cdffc33eb908a15631dc208253763 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 14:07:40 -0400 Subject: [PATCH 16/41] Generalized handler to allow it to be used with any arbitrary class structure. DocumentedGATKFeature now includes a field for the group name. Build.xml works with public / private now. --- build.xml | 3 ++- .../broadinstitute/sting/gatk/walkers/Walker.java | 4 ++-- .../sting/utils/help/DocumentedGATKFeature.java | 3 ++- .../utils/help/DocumentedGATKFeatureHandler.java | 10 +++++++++- .../broadinstitute/sting/utils/help/GATKDoclet.java | 4 +++- ...andler.java => GenericDocumentationHandler.java} | 13 +++++-------- ...ex.template.html => generic.index.template.html} | 0 .../{walker.template.html => generic.template.html} | 0 8 files changed, 23 insertions(+), 14 deletions(-) rename public/java/src/org/broadinstitute/sting/utils/help/{WalkerDocumentationHandler.java => GenericDocumentationHandler.java} (94%) rename settings/helpTemplates/{walker.index.template.html => generic.index.template.html} (100%) rename settings/helpTemplates/{walker.template.html => generic.template.html} (100%) diff --git a/build.xml b/build.xml index dcaafe322..51f39f7a3 100644 --- a/build.xml +++ b/build.xml @@ -457,8 +457,9 @@ - + diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java index 24813f4c6..9f26c9286 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java @@ -32,7 +32,7 @@ import org.broadinstitute.sting.utils.GenomeLoc; import org.broadinstitute.sting.utils.baq.BAQ; import org.broadinstitute.sting.utils.collections.Pair; import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; -import org.broadinstitute.sting.utils.help.WalkerDocumentationHandler; +import org.broadinstitute.sting.utils.help.GenericDocumentationHandler; import java.util.List; @@ -46,7 +46,7 @@ import java.util.List; @ReadFilters(MalformedReadFilter.class) @PartitionBy(PartitionType.NONE) @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT) -@DocumentedGATKFeature( handler = WalkerDocumentationHandler.class ) +@DocumentedGATKFeature( groupName = "GATK walkers" ) public abstract class Walker { final protected static Logger logger = Logger.getLogger(Walker.class); private GenomeAnalysisEngine toolkit; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java index 02a7707e6..a63869cad 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java @@ -36,5 +36,6 @@ import java.lang.annotation.*; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface DocumentedGATKFeature { - Class handler(); + String groupName(); + Class handler() default GenericDocumentationHandler.class; } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index e459e1543..fe3e56efe 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -49,6 +49,7 @@ import java.util.Map; */ public abstract class DocumentedGATKFeatureHandler { private GATKDoclet doclet; + private String groupName; protected RootDoc getRootDoc() { return this.doclet.rootDoc; @@ -64,7 +65,14 @@ public abstract class DocumentedGATKFeatureHandler { return ResourceBundleExtractorDoclet.getClassName(doc).replace(".", "_") + ".html"; } - public abstract String getGroupName(); + final public String getGroupName() { + return groupName; + } + + final public void setGroupName(String groupName) { + this.groupName = groupName; + } + public abstract String getTemplateName(ClassDoc doc) throws IOException; public abstract GATKDoclet.DocumentationData processOne(ClassDoc doc); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index e155923a5..81eaf632c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -110,6 +110,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { List indexData = new ArrayList(); for ( ClassDoc doc : rootDoc.classes() ) { + System.out.printf("Considering %s%n", doc); DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc); if ( handler != null && handler.shouldBeProcessed(doc) ) { DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc); @@ -132,6 +133,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) { DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class); DocumentedGATKFeatureHandler handler = feature.handler().newInstance(); + handler.setGroupName(feature.groupName()); handler.setDoclet(this); return handler; } else { @@ -152,7 +154,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { private void processIndex(Configuration cfg, List indexData) throws IOException { /* Get or create a template */ - Template temp = cfg.getTemplate("walker.index.template.html"); + Template temp = cfg.getTemplate("generic.index.template.html"); /* Merge data-model with template */ Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java similarity index 94% rename from public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java rename to public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index f9109db72..cab6c327d 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/WalkerDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -48,7 +48,7 @@ import java.util.Map; /** * */ -public class WalkerDocumentationHandler extends DocumentedGATKFeatureHandler { +public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { @Override public boolean shouldBeProcessed(ClassDoc doc) { try { @@ -59,17 +59,14 @@ public class WalkerDocumentationHandler extends DocumentedGATKFeatureHandler { } } - @Override - public String getGroupName() { return "GATK Walkers"; } - @Override public String getTemplateName(ClassDoc doc) throws IOException { - return "walker.template.html"; + return "generic.template.html"; } @Override public GATKDoclet.DocumentationData processOne(ClassDoc doc) { - System.out.printf("Walker class %s%n", doc); + System.out.printf("%s class %s%n", getGroupName(), doc); Map root = buildWalkerDataModel(doc); // Create the root hash return new GATKDoclet.DocumentationData(doc.name(), (String)root.get("summary"), root); } @@ -136,9 +133,9 @@ public class WalkerDocumentationHandler extends DocumentedGATKFeatureHandler { } private FieldDoc getFieldDoc(ClassDoc classDoc, String name, boolean primary) { - System.out.printf("Looking for %s in %s%n", name, classDoc.name()); + //System.out.printf("Looking for %s in %s%n", name, classDoc.name()); for ( FieldDoc fieldDoc : classDoc.fields(false) ) { - System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); + //System.out.printf("fieldDoc " + fieldDoc + " name " + fieldDoc.name()); if ( fieldDoc.name().equals(name) ) return fieldDoc; diff --git a/settings/helpTemplates/walker.index.template.html b/settings/helpTemplates/generic.index.template.html similarity index 100% rename from settings/helpTemplates/walker.index.template.html rename to settings/helpTemplates/generic.index.template.html diff --git a/settings/helpTemplates/walker.template.html b/settings/helpTemplates/generic.template.html similarity index 100% rename from settings/helpTemplates/walker.template.html rename to settings/helpTemplates/generic.template.html From 28b9432d2609b565abf9a912e3d6d9cab036f024 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Fri, 22 Jul 2011 16:09:21 -0400 Subject: [PATCH 17/41] Docs for read filters, the engine, and the UserExceptions. --- .../sting/utils/help/HelpUtils.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java diff --git a/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java b/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java new file mode 100644 index 000000000..dc6e6a533 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java @@ -0,0 +1,52 @@ +package org.broadinstitute.sting.utils.help; + +import com.sun.javadoc.FieldDoc; +import com.sun.javadoc.PackageDoc; +import com.sun.javadoc.ProgramElementDoc; + +import java.lang.reflect.Field; + +public class HelpUtils { + protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) { + for (Class interfaceClass : interfaceClasses) + if (assignableToClass(classDoc, interfaceClass, false)) + return true; + return false; + } + + protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { + try { + Class type = getClassForDoc(classDoc); + return lhsClass.isAssignableFrom(type) && (!requireConcrete || JVMUtils.isConcrete(type)); + } catch (Throwable t) { + // Ignore errors. + return false; + } + } + + protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException { + return Class.forName(getClassName(doc)); + } + + protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { + try { + Class clazz = getClassForDoc(fieldDoc.containingClass()); + return JVMUtils.findField(clazz, fieldDoc.name()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + } + + /** + * Reconstitute the class name from the given class JavaDoc object. + * + * @param doc the Javadoc model for the given class. + * @return The (string) class name of the given class. + */ + protected static String getClassName(ProgramElementDoc doc) { + PackageDoc containingPackage = doc.containingPackage(); + return containingPackage.name().length() > 0 ? + String.format("%s.%s", containingPackage.name(), doc.name()) : + String.format("%s", doc.name()); + } +} \ No newline at end of file From f0a6dd27a182d3ec5c290f2d942b0fa2d15d1caf Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sat, 23 Jul 2011 12:59:37 -0400 Subject: [PATCH 18/41] Renaming the plot output directory names. --- .../sting/queue/qscripts/RecalibrateBaseQualities.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/RecalibrateBaseQualities.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/RecalibrateBaseQualities.scala index f8218148e..cbe53db8d 100755 --- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/RecalibrateBaseQualities.scala +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/RecalibrateBaseQualities.scala @@ -42,8 +42,8 @@ class RecalibrateBaseQualities extends QScript { val recalFile1: File = swapExt(bam, ".bam", ".recal1.csv") val recalFile2: File = swapExt(bam, ".bam", ".recal2.csv") val recalBam: File = swapExt(bam, ".bam", ".recal.bam") - val path1: String = bam + ".before" - val path2: String = bam + ".after" + val path1: String = recalBam + ".before" + val path2: String = recalBam + ".after" add(cov(bam, recalFile1), recal(bam, recalFile1, recalBam), From 1f8fc4a0410ec2cd054e9534b1e597e8af495d22 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Sat, 23 Jul 2011 14:40:29 -0400 Subject: [PATCH 19/41] Added edu.mit.broad.picard to Queue packaged jar. Added hack to workaround EDU.oswego namespace and staging on mac case-insensitive filesystems. --- build.xml | 6 ++++++ public/packages/Queue.xml | 1 + 2 files changed, 7 insertions(+) diff --git a/build.xml b/build.xml index fe4c7a3f4..874a003eb 100644 --- a/build.xml +++ b/build.xml @@ -957,6 +957,12 @@ + + diff --git a/public/packages/Queue.xml b/public/packages/Queue.xml index 58da4398e..589cb45f5 100644 --- a/public/packages/Queue.xml +++ b/public/packages/Queue.xml @@ -41,6 +41,7 @@ + From e262f4e10b9f04653c6207e21d7e8537b14f3ed3 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 23 Jul 2011 20:00:35 -0400 Subject: [PATCH 21/41] gatkdoc now generalized to use @Annotation. Multiple subsystems now use annotation to receive docs Index expanded to use summary() annotation field UserExceptions, ReadFilters, GATK engine all use the system to generate docs Doclet expanded to handle lots of new cases --- build.xml | 16 ++++-- .../sting/gatk/CommandLineGATK.java | 3 ++ .../sting/gatk/filters/ReadFilter.java | 3 ++ .../sting/gatk/walkers/ReadFilters.java | 1 + .../sting/gatk/walkers/Walker.java | 4 +- .../sting/utils/exceptions/UserException.java | 4 ++ .../utils/help/DocumentedGATKFeature.java | 2 + .../help/DocumentedGATKFeatureHandler.java | 26 ++++------ .../sting/utils/help/GATKDoclet.java | 43 +++++++++++----- .../help/GenericDocumentationHandler.java | 11 ++-- .../sting/utils/help/HelpUtils.java | 25 +++++++++ .../help/ResourceBundleExtractorDoclet.java | 51 +------------------ .../helpTemplates/generic.index.template.html | 9 ++-- 13 files changed, 105 insertions(+), 93 deletions(-) diff --git a/build.xml b/build.xml index 51f39f7a3..6c1090ea4 100644 --- a/build.xml +++ b/build.xml @@ -79,6 +79,17 @@ + + + + + + + + + + + @@ -457,9 +468,8 @@ - - @@ -472,7 +482,7 @@ additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> - + diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index 3726e8e02..f8a2de316 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -50,6 +50,9 @@ import java.util.*; * gatk all the parsed out information. Pretty much anything dealing with the underlying system should go here, * the gatk engine should deal with any data related information. */ +@DocumentedGATKFeature( + groupName = "GATK Engine", + summary = "Features and arguments for the GATK engine itself, available to all walkers." ) public class CommandLineGATK extends CommandLineExecutable { @Argument(fullName = "analysis_type", shortName = "T", doc = "Type of analysis to run") private String analysisName = null; diff --git a/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java b/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java index 003d9cd42..bf3ce352a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java +++ b/public/java/src/org/broadinstitute/sting/gatk/filters/ReadFilter.java @@ -7,6 +7,9 @@ import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; /** * A SamRecordFilter that also depends on the header. */ +@DocumentedGATKFeature( + groupName = "Read filters", + summary = "GATK Engine arguments that filter or transfer incoming SAM/BAM data files" ) public abstract class ReadFilter implements SamRecordFilter { /** * Sets the header for use by this filter. diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/ReadFilters.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/ReadFilters.java index ff3b6d82f..5f11686a1 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/ReadFilters.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/ReadFilters.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.walkers; import net.sf.picard.filter.SamRecordFilter; +import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; import java.lang.annotation.*; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java index 9f26c9286..a5d2cd5b3 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java @@ -46,7 +46,9 @@ import java.util.List; @ReadFilters(MalformedReadFilter.class) @PartitionBy(PartitionType.NONE) @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT) -@DocumentedGATKFeature( groupName = "GATK walkers" ) +@DocumentedGATKFeature( + groupName = "GATK walkers", + summary = "General tools available for running on the command line as part of the GATK package" ) public abstract class Walker { final protected static Logger logger = Logger.getLogger(Walker.class); private GenomeAnalysisEngine toolkit; diff --git a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java index 7eab6f6c9..3c3299ff5 100755 --- a/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java +++ b/public/java/src/org/broadinstitute/sting/utils/exceptions/UserException.java @@ -29,6 +29,7 @@ import net.sf.samtools.SAMRecord; import net.sf.samtools.SAMSequenceDictionary; import net.sf.samtools.SAMSequenceRecord; import org.broadinstitute.sting.utils.GenomeLoc; +import org.broadinstitute.sting.utils.help.DocumentedGATKFeature; import org.broadinstitute.sting.utils.variantcontext.VariantContext; import java.io.File; @@ -43,6 +44,9 @@ import java.util.Arrays; * Date: Sep 3, 2010 * Time: 2:24:09 PM */ +@DocumentedGATKFeature( + groupName = "User exceptions", + summary = "Exceptions caused by incorrect user behavior, such as bad files, bad arguments, etc." ) public class UserException extends ReviewedStingException { public UserException(String msg) { super(msg); } public UserException(String msg, Throwable e) { super(msg, e); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java index a63869cad..689d1b7ad 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java @@ -36,6 +36,8 @@ import java.lang.annotation.*; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface DocumentedGATKFeature { + boolean enable() default true; String groupName(); + String summary() default ""; Class handler() default GenericDocumentationHandler.class; } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index fe3e56efe..ddcc12ab3 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -25,24 +25,9 @@ package org.broadinstitute.sting.utils.help; import com.sun.javadoc.ClassDoc; -import com.sun.javadoc.FieldDoc; import com.sun.javadoc.RootDoc; -import com.sun.javadoc.Tag; -import freemarker.template.Configuration; -import freemarker.template.DefaultObjectWrapper; -import freemarker.template.Template; -import freemarker.template.TemplateException; -import org.broadinstitute.sting.commandline.*; -import org.broadinstitute.sting.gatk.CommandLineGATK; -import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; /** * @@ -50,6 +35,7 @@ import java.util.Map; public abstract class DocumentedGATKFeatureHandler { private GATKDoclet doclet; private String groupName; + private DocumentedGATKFeature annotation; protected RootDoc getRootDoc() { return this.doclet.rootDoc; @@ -59,10 +45,18 @@ public abstract class DocumentedGATKFeatureHandler { this.doclet = doclet; } + public DocumentedGATKFeature getAnnotation() { + return annotation; + } + + public void setAnnotation(DocumentedGATKFeature annotation) { + this.annotation = annotation; + } + public boolean shouldBeProcessed(ClassDoc doc) { return true; } public String getDestinationFilename(ClassDoc doc) { - return ResourceBundleExtractorDoclet.getClassName(doc).replace(".", "_") + ".html"; + return HelpUtils.getClassName(doc).replace(".", "_") + ".html"; } final public String getGroupName() { diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 81eaf632c..ae8ab2c06 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -109,16 +109,18 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { cfg.setObjectWrapper(new DefaultObjectWrapper()); List indexData = new ArrayList(); + Set docFeatures = new HashSet(); for ( ClassDoc doc : rootDoc.classes() ) { System.out.printf("Considering %s%n", doc); DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc); if ( handler != null && handler.shouldBeProcessed(doc) ) { DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc); + docFeatures.add(handler.getAnnotation()); indexData.add(docData); } } - processIndex(cfg, indexData); + processIndex(cfg, indexData, new ArrayList(docFeatures)); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -129,13 +131,19 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { private DocumentedGATKFeatureHandler getHandlerForClassDoc(ClassDoc doc) { try { // todo -- what do I need the ? extends Object to pass the compiler? - Class docClass = ResourceBundleExtractorDoclet.getClassForDoc(doc); + Class docClass = HelpUtils.getClassForDoc(doc); if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) { DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class); - DocumentedGATKFeatureHandler handler = feature.handler().newInstance(); - handler.setGroupName(feature.groupName()); - handler.setDoclet(this); - return handler; + if ( feature.enable() ) { + DocumentedGATKFeatureHandler handler = feature.handler().newInstance(); + handler.setGroupName(feature.groupName()); + handler.setDoclet(this); + handler.setAnnotation(feature); + return handler; + } else { + logger.info("Skipping disabled Documentation for " + doc); + return null; + } } else { return null; // not annotated so it shouldn't be documented } @@ -152,21 +160,21 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - private void processIndex(Configuration cfg, List indexData) throws IOException { + private void processIndex(Configuration cfg, List indexData, List docFeatures) throws IOException { /* Get or create a template */ Template temp = cfg.getTemplate("generic.index.template.html"); /* Merge data-model with template */ Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); try { - temp.process(groupIndexData(indexData), out); + temp.process(groupIndexData(indexData, docFeatures), out); out.flush(); } catch ( TemplateException e ) { throw new ReviewedStingException("Failed to create GATK documentation", e); } } - private Map groupIndexData(List indexData) { + private Map groupIndexData(List indexData, List docFeatures) { // // root -> data -> { summary -> y, filename -> z }, etc // -> groups -> group1, group2, etc. @@ -175,19 +183,28 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Collections.sort(indexData); List> data = new ArrayList>(); - Set groups = new HashSet(); - for ( DocumentationData indexDatum : indexData ) { data.add(indexDatum.toMap()); - groups.add(indexDatum.group); + } + + List> groups = new ArrayList>(); + for ( DocumentedGATKFeature feature : docFeatures ) { + groups.add(toMap(feature)); } root.put("data", data); - root.put("groups", new ArrayList(groups)); + root.put("groups", groups); return root; } + private static final Map toMap(DocumentedGATKFeature annotation) { + Map root = new HashMap(); + root.put("name", annotation.groupName()); + root.put("summary", annotation.summary()); + return root; + } + private DocumentationData processDocumentationWithHandler(Configuration cfg, DocumentedGATKFeatureHandler handler, ClassDoc doc) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index cab6c327d..a4e3e5031 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -26,12 +26,7 @@ package org.broadinstitute.sting.utils.help; import com.sun.javadoc.ClassDoc; import com.sun.javadoc.FieldDoc; -import com.sun.javadoc.RootDoc; import com.sun.javadoc.Tag; -import freemarker.template.Configuration; -import freemarker.template.DefaultObjectWrapper; -import freemarker.template.Template; -import freemarker.template.TemplateException; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.utils.Utils; @@ -52,7 +47,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { @Override public boolean shouldBeProcessed(ClassDoc doc) { try { - Class type = ResourceBundleExtractorDoclet.getClassForDoc(doc); + Class type = HelpUtils.getClassForDoc(doc); return JVMUtils.isConcrete(type); } catch ( ClassNotFoundException e ) { return false; @@ -98,7 +93,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { args.put("hidden", new ArrayList()); args.put("depreciated", new ArrayList()); try { - for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(ResourceBundleExtractorDoclet.getClassForDoc(classdoc)) ) { + for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(HelpUtils.getClassForDoc(classdoc)) ) { ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? @@ -139,7 +134,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { if ( fieldDoc.name().equals(name) ) return fieldDoc; - Field field = ResourceBundleExtractorDoclet.getFieldForFieldDoc(fieldDoc); + Field field = HelpUtils.getFieldForFieldDoc(fieldDoc); if ( field.isAnnotationPresent(ArgumentCollection.class) ) { ClassDoc typeDoc = getRootDoc().classNamed(fieldDoc.type().qualifiedTypeName()); if ( typeDoc == null ) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java b/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java index dc6e6a533..4527c6afe 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/HelpUtils.java @@ -1,8 +1,33 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.utils.help; import com.sun.javadoc.FieldDoc; import com.sun.javadoc.PackageDoc; import com.sun.javadoc.ProgramElementDoc; +import org.broadinstitute.sting.utils.classloader.JVMUtils; import java.lang.reflect.Field; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java index 140e06481..a28a7bcee 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/ResourceBundleExtractorDoclet.java @@ -28,12 +28,8 @@ package org.broadinstitute.sting.utils.help; import com.sun.javadoc.*; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.classloader.JVMUtils; -import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import sun.tools.java.ClassNotFound; import java.io.*; -import java.lang.reflect.Field; import java.util.*; /** @@ -112,7 +108,7 @@ public class ResourceBundleExtractorDoclet { if(isRequiredJavadocMissing(currentClass) && isWalker(currentClass)) undocumentedWalkers.add(currentClass.name()); - renderHelpText(getClassName(currentClass),currentClass); + renderHelpText(HelpUtils.getClassName(currentClass),currentClass); } for(PackageDoc currentPackage: packages) @@ -177,50 +173,7 @@ public class ResourceBundleExtractorDoclet { * @return True if the class of the given name is a walker. False otherwise. */ protected static boolean isWalker(ClassDoc classDoc) { - return assignableToClass(classDoc, Walker.class, true); - } - - protected static boolean implementsInterface(ProgramElementDoc classDoc, Class... interfaceClasses) { - for ( Class interfaceClass : interfaceClasses ) - if ( assignableToClass(classDoc, interfaceClass, false) ) - return true; - return false; - } - - protected static boolean assignableToClass(ProgramElementDoc classDoc, Class lhsClass, boolean requireConcrete) { - try { - Class type = getClassForDoc(classDoc); - return lhsClass.isAssignableFrom(type) && (! requireConcrete || JVMUtils.isConcrete(type)); - } - catch(Throwable t) { - // Ignore errors. - return false; - } - } - - protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException { - return Class.forName(getClassName(doc)); - } - - protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { - try { - Class clazz = getClassForDoc(fieldDoc.containingClass()); - return JVMUtils.findField(clazz, fieldDoc.name()); - } catch ( ClassNotFoundException e ) { - throw new RuntimeException(e); - } - } - - /** - * Reconstitute the class name from the given class JavaDoc object. - * @param doc the Javadoc model for the given class. - * @return The (string) class name of the given class. - */ - protected static String getClassName(ProgramElementDoc doc) { - PackageDoc containingPackage = doc.containingPackage(); - return containingPackage.name().length() > 0 ? - String.format("%s.%s",containingPackage.name(),doc.name()) : - String.format("%s",doc.name()); + return HelpUtils.assignableToClass(classDoc, Walker.class, true); } /** diff --git a/settings/helpTemplates/generic.index.template.html b/settings/helpTemplates/generic.index.template.html index 83020a6e0..b494dc582 100644 --- a/settings/helpTemplates/generic.index.template.html +++ b/settings/helpTemplates/generic.index.template.html @@ -1,12 +1,15 @@ <#macro emitGroup group> -

${group}

+

${group.name}

+

+ ${group.summary} +

<#list data as datum> - <#if datum.group == group> + <#if datum.group == group.name> @@ -21,7 +24,7 @@ GATK documentation index -

GATK documentation index

+

GATK documentation index

<#list groups as group> <@emitGroup group=group/> From 7420ed098e722610dbf9dcac00320a30ca1ba073 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 23 Jul 2011 22:07:30 -0400 Subject: [PATCH 22/41] Semi-working version of extraDocs tag in annotation to refer to one capability being accessible in another Required a significant refactoring of the GATKDoclet, which now has a unified place where the ClassDoc, class, annotation, and handler are all stored together. --- build.xml | 2 +- .../sting/gatk/CommandLineGATK.java | 11 +- .../sting/gatk/walkers/Walker.java | 4 +- .../utils/help/DocumentedGATKFeature.java | 9 +- .../help/DocumentedGATKFeatureHandler.java | 21 +-- .../sting/utils/help/GATKDoclet.java | 144 +++++++++++------- .../help/GenericDocumentationHandler.java | 32 ++-- settings/helpTemplates/generic.template.html | 61 +++++--- 8 files changed, 163 insertions(+), 121 deletions(-) diff --git a/build.xml b/build.xml index b43f3421e..ed7802b1d 100644 --- a/build.xml +++ b/build.xml @@ -468,7 +468,7 @@ - diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index f8a2de316..905e75584 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -30,6 +30,7 @@ import org.broadinstitute.sting.commandline.Argument; import org.broadinstitute.sting.commandline.ArgumentCollection; import org.broadinstitute.sting.commandline.CommandLineProgram; import org.broadinstitute.sting.gatk.arguments.GATKArgumentCollection; +import org.broadinstitute.sting.gatk.filters.ReadFilter; import org.broadinstitute.sting.gatk.walkers.Attribution; import org.broadinstitute.sting.gatk.walkers.Walker; import org.broadinstitute.sting.utils.exceptions.UserException; @@ -40,12 +41,8 @@ import org.broadinstitute.sting.utils.text.TextFormattingUtils; import java.util.*; /** - * @author aaron - * @version 1.0 - * @date May 8, 2009 - *

- * Class CommandLineGATK - *

+ * The GATK engine itself. Manages map/reduce data access and runs walkers. + * * We run command line GATK programs using this class. It gets the command line args, parses them, and hands the * gatk all the parsed out information. Pretty much anything dealing with the underlying system should go here, * the gatk engine should deal with any data related information. @@ -53,6 +50,8 @@ import java.util.*; @DocumentedGATKFeature( groupName = "GATK Engine", summary = "Features and arguments for the GATK engine itself, available to all walkers." ) +//, +// extraDocs = { ReadFilter.class, UserException.class }) public class CommandLineGATK extends CommandLineExecutable { @Argument(fullName = "analysis_type", shortName = "T", doc = "Type of analysis to run") private String analysisName = null; diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java index a5d2cd5b3..9e261a0b1 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/Walker.java @@ -26,6 +26,7 @@ package org.broadinstitute.sting.gatk.walkers; import org.apache.log4j.Logger; +import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.gatk.GenomeAnalysisEngine; import org.broadinstitute.sting.gatk.filters.MalformedReadFilter; import org.broadinstitute.sting.utils.GenomeLoc; @@ -48,7 +49,8 @@ import java.util.List; @BAQMode(QualityMode = BAQ.QualityMode.OVERWRITE_QUALS, ApplicationTime = BAQ.ApplicationTime.ON_INPUT) @DocumentedGATKFeature( groupName = "GATK walkers", - summary = "General tools available for running on the command line as part of the GATK package" ) + summary = "General tools available for running on the command line as part of the GATK package", + extraDocs = {CommandLineGATK.class}) public abstract class Walker { final protected static Logger logger = Logger.getLogger(Walker.class); private GenomeAnalysisEngine toolkit; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java index 689d1b7ad..710503ca8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeature.java @@ -36,8 +36,9 @@ import java.lang.annotation.*; @Retention(RetentionPolicy.RUNTIME) @Target(ElementType.TYPE) public @interface DocumentedGATKFeature { - boolean enable() default true; - String groupName(); - String summary() default ""; - Class handler() default GenericDocumentationHandler.class; + public boolean enable() default true; + public String groupName(); + public String summary() default ""; + public Class handler() default GenericDocumentationHandler.class; + public Class[] extraDocs() default {}; } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index ddcc12ab3..42bd41905 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -28,14 +28,13 @@ import com.sun.javadoc.ClassDoc; import com.sun.javadoc.RootDoc; import java.io.*; +import java.util.Map; /** * */ public abstract class DocumentedGATKFeatureHandler { private GATKDoclet doclet; - private String groupName; - private DocumentedGATKFeature annotation; protected RootDoc getRootDoc() { return this.doclet.rootDoc; @@ -45,28 +44,12 @@ public abstract class DocumentedGATKFeatureHandler { this.doclet = doclet; } - public DocumentedGATKFeature getAnnotation() { - return annotation; - } - - public void setAnnotation(DocumentedGATKFeature annotation) { - this.annotation = annotation; - } - public boolean shouldBeProcessed(ClassDoc doc) { return true; } public String getDestinationFilename(ClassDoc doc) { return HelpUtils.getClassName(doc).replace(".", "_") + ".html"; } - final public String getGroupName() { - return groupName; - } - - final public void setGroupName(String groupName) { - this.groupName = groupName; - } - public abstract String getTemplateName(ClassDoc doc) throws IOException; - public abstract GATKDoclet.DocumentationData processOne(ClassDoc doc); + public abstract void processOne(GATKDoclet.DocWorkUnit toProcess, Map all); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index ae8ab2c06..70d455794 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -42,25 +42,34 @@ import java.util.*; public class GATKDoclet extends ResourceBundleExtractorDoclet { final protected static Logger logger = Logger.getLogger(GATKDoclet.class); - public static class DocumentationData implements Comparable { - String name, summary, filename; - Map forTemplate; - String group; + public static class DocWorkUnit implements Comparable { + // known at the start + String name, filename, group; + DocumentedGATKFeatureHandler handler; + ClassDoc classDoc; + Class clazz; + DocumentedGATKFeature annotation; - public DocumentationData(String name, String summary, Map forTemplate) { + // set by the handler + String summary; + Map forTemplate; + + public DocWorkUnit(DocumentedGATKFeature annotation, String name, String filename, String group, + DocumentedGATKFeatureHandler handler, ClassDoc classDoc, Class clazz) { + this.annotation = annotation; this.name = name; + this.filename = filename; + this.group = group; + this.handler = handler; + this.classDoc = classDoc; + this.clazz = clazz; + } + + public void setHandlerContent(String summary, Map forTemplate) { this.summary = summary; this.forTemplate = forTemplate; } - public void setGroup(String group) { - this.group = group; - } - - public void setFilename(String filename) { - this.filename = filename; - } - public Map toMap() { Map data = new HashMap(); data.put("name", name); @@ -70,7 +79,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return data; } - public int compareTo(DocumentationData other) { + public int compareTo(DocWorkUnit other) { return this.name.compareTo(other.name); } } @@ -93,6 +102,26 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return ResourceBundleExtractorDoclet.optionLength(option); } + public Map workUnits() { + Map m = new HashMap(); + + for ( ClassDoc doc : rootDoc.classes() ) { + System.out.printf("Considering %s%n", doc); + Class clazz = getClassForClassDoc(doc); + DocumentedGATKFeature feature = getFeatureForClassDoc(doc); + DocumentedGATKFeatureHandler handler = createHandler(doc, feature); + if ( handler != null && handler.shouldBeProcessed(doc) ) { + String filename = handler.getDestinationFilename(doc); + DocWorkUnit unit = new DocWorkUnit(feature, + doc.name(), filename, feature.groupName(), + handler, doc, clazz ); + m.put(clazz, unit); + } + } + + return m; + } + @Override protected void processDocs(RootDoc rootDoc, PrintStream ignore) { // setup the global access to the root @@ -108,19 +137,12 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { // Specify how templates will see the data-model. This is an advanced topic... cfg.setObjectWrapper(new DefaultObjectWrapper()); - List indexData = new ArrayList(); - Set docFeatures = new HashSet(); - for ( ClassDoc doc : rootDoc.classes() ) { - System.out.printf("Considering %s%n", doc); - DocumentedGATKFeatureHandler handler = getHandlerForClassDoc(doc); - if ( handler != null && handler.shouldBeProcessed(doc) ) { - DocumentationData docData = processDocumentationWithHandler(cfg, handler, doc); - docFeatures.add(handler.getAnnotation()); - indexData.add(docData); - } + Map workUnitMap = workUnits(); + for ( DocWorkUnit workUnit : workUnitMap.values() ) { + processDocWorkUnit(cfg, workUnit, workUnitMap); } - processIndex(cfg, indexData, new ArrayList(docFeatures)); + processIndex(cfg, new ArrayList(workUnitMap.values())); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -128,53 +150,66 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - private DocumentedGATKFeatureHandler getHandlerForClassDoc(ClassDoc doc) { + private DocumentedGATKFeatureHandler createHandler(ClassDoc doc, DocumentedGATKFeature feature) { try { - // todo -- what do I need the ? extends Object to pass the compiler? - Class docClass = HelpUtils.getClassForDoc(doc); - if ( docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) { - DocumentedGATKFeature feature = docClass.getAnnotation(DocumentedGATKFeature.class); + if ( feature != null ) { if ( feature.enable() ) { DocumentedGATKFeatureHandler handler = feature.handler().newInstance(); - handler.setGroupName(feature.groupName()); handler.setDoclet(this); - handler.setAnnotation(feature); return handler; } else { logger.info("Skipping disabled Documentation for " + doc); - return null; } - } else { - return null; // not annotated so it shouldn't be documented } - } catch ( ClassNotFoundException e) { - //logger.warn("Couldn't find class for ClassDoc " + doc); - // we got a classdoc for a class we can't find. Maybe in a library or something - return null; } catch ( IllegalAccessException e) { throw new RuntimeException(e); // the constructor is now private -- this is an error } catch ( InstantiationException e) { throw new RuntimeException(e); // the constructor is now private -- this is an error + } + + return null; + } + + private DocumentedGATKFeature getFeatureForClassDoc(ClassDoc doc) { + // todo -- what do I need the ? extends Object to pass the compiler? + Class docClass = getClassForClassDoc(doc); + if ( docClass != null && docClass.isAnnotationPresent(DocumentedGATKFeature.class) ) { + return docClass.getAnnotation(DocumentedGATKFeature.class); + } else { + return null; // not annotated so it shouldn't be documented + } + } + + private Class getClassForClassDoc(ClassDoc doc) { + try { + // todo -- what do I need the ? extends Object to pass the compiler? + return (Class)HelpUtils.getClassForDoc(doc); + } catch ( ClassNotFoundException e) { + //logger.warn("Couldn't find class for ClassDoc " + doc); + // we got a classdoc for a class we can't find. Maybe in a library or something + return null; + } catch ( NoClassDefFoundError e ) { + return null; } catch ( UnsatisfiedLinkError e) { return null; // naughty BWA bindings } } - private void processIndex(Configuration cfg, List indexData, List docFeatures) throws IOException { + private void processIndex(Configuration cfg, List indexData) throws IOException { /* Get or create a template */ Template temp = cfg.getTemplate("generic.index.template.html"); /* Merge data-model with template */ Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); try { - temp.process(groupIndexData(indexData, docFeatures), out); + temp.process(groupIndexData(indexData), out); out.flush(); } catch ( TemplateException e ) { throw new ReviewedStingException("Failed to create GATK documentation", e); } } - private Map groupIndexData(List indexData, List docFeatures) { + private Map groupIndexData(List indexData) { // // root -> data -> { summary -> y, filename -> z }, etc // -> groups -> group1, group2, etc. @@ -182,9 +217,11 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Collections.sort(indexData); + Set docFeatures = new HashSet(); List> data = new ArrayList>(); - for ( DocumentationData indexDatum : indexData ) { - data.add(indexDatum.toMap()); + for ( DocWorkUnit workUnit : indexData ) { + data.add(workUnit.toMap()); + docFeatures.add(workUnit.annotation); } List> groups = new ArrayList>(); @@ -205,30 +242,23 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - private DocumentationData processDocumentationWithHandler(Configuration cfg, - DocumentedGATKFeatureHandler handler, - ClassDoc doc) + private void processDocWorkUnit(Configuration cfg, DocWorkUnit unit, Map all) throws IOException { - System.out.printf("Processing documentation for class %s%n", doc); + System.out.printf("Processing documentation for class %s%n", unit.classDoc); - DocumentationData docData = handler.processOne(doc); - docData.setGroup(handler.getGroupName()); + unit.handler.processOne(unit, all); // Get or create a template - Template temp = cfg.getTemplate(handler.getTemplateName(doc)); + Template temp = cfg.getTemplate(unit.handler.getTemplateName(unit.classDoc)); // Merge data-model with template - String filename = handler.getDestinationFilename(doc); - docData.setFilename(filename); - File outputPath = new File("testdoc/" + filename); + File outputPath = new File("testdoc/" + unit.filename); try { Writer out = new OutputStreamWriter(new FileOutputStream(outputPath)); - temp.process(docData.forTemplate, out); + temp.process(unit.forTemplate, out); out.flush(); } catch ( TemplateException e ) { throw new ReviewedStingException("Failed to create GATK documentation", e); } - - return docData; } } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index a4e3e5031..d85e86bfe 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -35,10 +35,7 @@ import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; /** * @@ -54,20 +51,16 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } } + @Override public String getTemplateName(ClassDoc doc) throws IOException { return "generic.template.html"; } @Override - public GATKDoclet.DocumentationData processOne(ClassDoc doc) { - System.out.printf("%s class %s%n", getGroupName(), doc); - Map root = buildWalkerDataModel(doc); // Create the root hash - return new GATKDoclet.DocumentationData(doc.name(), (String)root.get("summary"), root); - } - - - private Map buildWalkerDataModel(ClassDoc classdoc) { + public void processOne(GATKDoclet.DocWorkUnit toProcess, Map all) { + System.out.printf("%s class %s%n", toProcess.group, toProcess.classDoc); + ClassDoc classdoc = toProcess.classDoc; Map root = new HashMap(); root.put("name", classdoc.name()); @@ -109,8 +102,21 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { throw new RuntimeException(e); } + List> extraDocsData = new ArrayList>(); + for ( Class extraDocClass : toProcess.annotation.extraDocs() ) { + final GATKDoclet.DocWorkUnit otherUnit = all.get(extraDocClass); + if ( otherUnit == null ) + throw new ReviewedStingException("Requested extraDocs for class without any documentation: " + extraDocClass); + extraDocsData.add( + new HashMap(){{ + put("filename", otherUnit.filename); + put("name", otherUnit.name);}}); + + } + root.put("extradocs", extraDocsData); + //System.out.printf("Root is %s%n", root); - return root; + toProcess.setHandlerContent(summaryBuilder.toString(), root); } protected ParsingEngine createStandardGATKParsingEngine() { diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index ecf24a56e..27c26749e 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -38,26 +38,47 @@

Description

${description} - <#-- Create the argument summary --> -

Arguments

-
Name Summary
${datum.name} ${datum.summary}
- - - - - - - <@argumentlist name="Required" myargs=arguments.required/> - <@argumentlist name="Optional" myargs=arguments.optional/> - <@argumentlist name="Hidden" myargs=arguments.hidden/> - <@argumentlist name="Depreciated" myargs=arguments.depreciated/> -
NameSynonymsTypeSummary
- - <#-- Create the argument details --> -

Argument details

- <#list arguments.all as arg> - <@argumentDetails arg=arg/> - + <#-- Create the argument summary --> + <#if arguments.all?size != 0> +
+

Feature specific arguments

+ + + + + + + + <@argumentlist name="Required" myargs=arguments.required/> + <@argumentlist name="Optional" myargs=arguments.optional/> + <@argumentlist name="Hidden" myargs=arguments.hidden/> + <@argumentlist name="Depreciated" myargs=arguments.depreciated/> +
NameSynonymsTypeSummary
+ + + <#-- Create references to related capabilities if appropriate --> + <#if extradocs?size != 0> +
+

Related capabilities

+ The arguments described in the entries below can be supplied to this tool to modify + its behavior. For example, the -L argument directs the GATK engine restricts processing + to specific genomic intervals. This capability is available to all GATK walkers. + + + + <#-- List all of the --> + <#if arguments.all?size != 0> +
+ <#-- Create the argument details --> +

Argument details

+ <#list arguments.all as arg> + <@argumentDetails arg=arg/> + + From 6b501e267bfb770f21f21048ac85551ee2ee3d8c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sat, 23 Jul 2011 22:15:01 -0400 Subject: [PATCH 23/41] Includes non-concrete classes in docs CommandLineGATK has extraDocs to ReadFilter and UserException as well --- .../broadinstitute/sting/gatk/CommandLineGATK.java | 5 ++--- .../utils/help/GenericDocumentationHandler.java | 13 +++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java index 905e75584..2af29ea70 100755 --- a/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java +++ b/public/java/src/org/broadinstitute/sting/gatk/CommandLineGATK.java @@ -49,9 +49,8 @@ import java.util.*; */ @DocumentedGATKFeature( groupName = "GATK Engine", - summary = "Features and arguments for the GATK engine itself, available to all walkers." ) -//, -// extraDocs = { ReadFilter.class, UserException.class }) + summary = "Features and arguments for the GATK engine itself, available to all walkers.", + extraDocs = { ReadFilter.class, UserException.class }) public class CommandLineGATK extends CommandLineExecutable { @Argument(fullName = "analysis_type", shortName = "T", doc = "Type of analysis to run") private String analysisName = null; diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index d85e86bfe..8b0a33887 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -43,12 +43,13 @@ import java.util.*; public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { @Override public boolean shouldBeProcessed(ClassDoc doc) { - try { - Class type = HelpUtils.getClassForDoc(doc); - return JVMUtils.isConcrete(type); - } catch ( ClassNotFoundException e ) { - return false; - } + return true; +// try { +// Class type = HelpUtils.getClassForDoc(doc); +// return JVMUtils.isConcrete(type); +// } catch ( ClassNotFoundException e ) { +// return false; +// } } From 7ffedf211ca8123db6d5cb64db526d1114d7fbe4 Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Sun, 24 Jul 2011 02:24:04 -0400 Subject: [PATCH 24/41] Contig comparator -- sorting contigs like Picard This is very useful if you want to output your text files or manipulate data in the usual chromosome ordering : 1 2 3 ... 21 22 X Y GL??? ... Just use this comparator in any SortedSet class constructor and your data will be sorted like in the BAM file. --- .../sting/utils/ContigComparator.java | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 public/java/src/org/broadinstitute/sting/utils/ContigComparator.java diff --git a/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java b/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java new file mode 100644 index 000000000..5e573418d --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java @@ -0,0 +1,57 @@ +package org.broadinstitute.sting.utils; + +import java.util.Comparator; +import java.util.Set; +import java.util.TreeSet; + +/** + * Created by IntelliJ IDEA. + * User: carneiro + * Date: 7/23/11 + * Time: 6:07 PM + * To change this template use File | Settings | File Templates. + */ +public class ContigComparator implements Comparator { + private Set specialChrs; + + public ContigComparator() { + specialChrs = new TreeSet(); + specialChrs.add("X"); + specialChrs.add("Y"); + } + + public int compare(String chr1, String chr2) { + if (chr1.equals(chr2)) + return 0; + + Integer x = convertStringWithoutException(chr1); + Integer y = convertStringWithoutException(chr2); + // both contigs are numbered + if (x != null && y != null) + return (x < y) ? -1:1; + + // both contigs are named + if (x == null && y == null) { + // both contigs are special contigs or neither contig is a special contigs + if (specialChrs.contains(chr1) && specialChrs.contains(chr2) || (!specialChrs.contains(chr1) && !specialChrs.contains(chr2))) + return chr1.compareTo(chr2); + // one contig is a special and the other is not special + if (specialChrs.contains(chr1)) + return -1; + return 1; + } + + // one contig is named the other is numbered + if (x != null) + return -1; + return 1; + } + + private Integer convertStringWithoutException(String contig) { + Integer x = null; + try { + x = Integer.decode(contig); + } catch (NumberFormatException n){} + return x; + } +} From d0ab6bf7a932bde2fc3e77be36f9e468882ddd95 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 09:56:17 -0400 Subject: [PATCH 26/41] Now links to sub and superclass documentation, where possible. --- .../help/DocumentedGATKFeatureHandler.java | 3 +- .../sting/utils/help/GATKDoclet.java | 23 ++++--- .../help/GenericDocumentationHandler.java | 62 ++++++++++++++++--- settings/helpTemplates/generic.template.html | 33 ++++++++-- 4 files changed, 101 insertions(+), 20 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index 42bd41905..f25a02bb5 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -29,6 +29,7 @@ import com.sun.javadoc.RootDoc; import java.io.*; import java.util.Map; +import java.util.Set; /** * @@ -51,5 +52,5 @@ public abstract class DocumentedGATKFeatureHandler { } public abstract String getTemplateName(ClassDoc doc) throws IOException; - public abstract void processOne(GATKDoclet.DocWorkUnit toProcess, Map all); + public abstract void processOne(GATKDoclet.DocWorkUnit toProcess, Set all); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 70d455794..aa5514dae 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -102,8 +102,8 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return ResourceBundleExtractorDoclet.optionLength(option); } - public Map workUnits() { - Map m = new HashMap(); + public Set workUnits() { + TreeSet m = new TreeSet(); for ( ClassDoc doc : rootDoc.classes() ) { System.out.printf("Considering %s%n", doc); @@ -115,7 +115,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { DocWorkUnit unit = new DocWorkUnit(feature, doc.name(), filename, feature.groupName(), handler, doc, clazz ); - m.put(clazz, unit); + m.add(unit); } } @@ -137,12 +137,12 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { // Specify how templates will see the data-model. This is an advanced topic... cfg.setObjectWrapper(new DefaultObjectWrapper()); - Map workUnitMap = workUnits(); - for ( DocWorkUnit workUnit : workUnitMap.values() ) { - processDocWorkUnit(cfg, workUnit, workUnitMap); + Set myWorkUnits = workUnits(); + for ( DocWorkUnit workUnit : myWorkUnits ) { + processDocWorkUnit(cfg, workUnit, myWorkUnits); } - processIndex(cfg, new ArrayList(workUnitMap.values())); + processIndex(cfg, new ArrayList(myWorkUnits)); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -242,7 +242,14 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - private void processDocWorkUnit(Configuration cfg, DocWorkUnit unit, Map all) + public final static DocWorkUnit findWorkUnitForClass(Class c, Set all) { + for ( final DocWorkUnit unit : all ) + if ( unit.clazz.equals(c) ) + return unit; + return null; + } + + private void processDocWorkUnit(Configuration cfg, DocWorkUnit unit, Set all) throws IOException { System.out.printf("Processing documentation for class %s%n", unit.classDoc); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 8b0a33887..cbe526793 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -41,6 +41,10 @@ import java.util.*; * */ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { + GATKDoclet.DocWorkUnit toProcess; + ClassDoc classdoc; + Set all; + @Override public boolean shouldBeProcessed(ClassDoc doc) { return true; @@ -59,11 +63,23 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } @Override - public void processOne(GATKDoclet.DocWorkUnit toProcess, Map all) { + public void processOne(GATKDoclet.DocWorkUnit toProcessArg, Set allArg) { + this.toProcess = toProcessArg; + this.all = allArg; + this.classdoc = toProcess.classDoc; + System.out.printf("%s class %s%n", toProcess.group, toProcess.classDoc); - ClassDoc classdoc = toProcess.classDoc; Map root = new HashMap(); + addHighLevelBindings(root); + addArgumentBindings(root); + addRelatedBindings(root); + + //System.out.printf("Root is %s%n", root); + toProcess.setHandlerContent((String)root.get("summary"), root); + } + + protected void addHighLevelBindings(Map root) { root.put("name", classdoc.name()); // Extract overrides from the doc tags. @@ -76,7 +92,9 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { for(Tag tag: classdoc.tags()) { root.put(tag.name(), tag.text()); } + } + protected void addArgumentBindings(Map root) { ParsingEngine parsingEngine = createStandardGATKParsingEngine(); Map> args = new HashMap>(); @@ -102,10 +120,14 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } catch ( ClassNotFoundException e ) { throw new RuntimeException(e); } + } + protected void addRelatedBindings(Map root) { List> extraDocsData = new ArrayList>(); - for ( Class extraDocClass : toProcess.annotation.extraDocs() ) { - final GATKDoclet.DocWorkUnit otherUnit = all.get(extraDocClass); + + // add in all of the explicitly related items + for ( final Class extraDocClass : toProcess.annotation.extraDocs() ) { + final GATKDoclet.DocWorkUnit otherUnit = GATKDoclet.findWorkUnitForClass(extraDocClass, all); if ( otherUnit == null ) throw new ReviewedStingException("Requested extraDocs for class without any documentation: " + extraDocClass); extraDocsData.add( @@ -114,10 +136,36 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { put("name", otherUnit.name);}}); } - root.put("extradocs", extraDocsData); - //System.out.printf("Root is %s%n", root); - toProcess.setHandlerContent(summaryBuilder.toString(), root); + List> hierarchyDocs = new ArrayList>(); + for (final GATKDoclet.DocWorkUnit other : all ) { + final String relation = classRelationship(toProcess.clazz, other.clazz); + if ( relation != null ) + hierarchyDocs.add( + new HashMap(){{ + put("filename", other.filename); + put("relation", relation); + put("name", other.name);}}); + + } + + root.put("relatedDocs", hierarchyDocs); + root.put("extradocs", extraDocsData); + } + + private static final String classRelationship(Class me, Class other) { + if ( other.equals(me) ) + // no circular references + return null; + else if ( other.isAssignableFrom(me) ) + // toProcess is a superclass of other.clazz + return "superclass"; + else if ( me.isAssignableFrom(other) ) + // toProcess inherits from other.clazz + return "subclass"; + else + return null; + } protected ParsingEngine createStandardGATKParsingEngine() { diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index 27c26749e..6540ba0e3 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -24,19 +24,36 @@ Details: ${arg.fulltext}
+<#macro relatedByType name type> + <#list relatedDocs as relatedDoc> + <#if relatedDoc.relation == type> +

${name}

+
    + <#list relatedDocs as relatedDoc> + <#if relatedDoc.relation == type> +
  • ${relatedDoc.name} is a ${relatedDoc.relation}
    • + + +
+ <#break> + + + + + ${name} documentation

${name}

-

Summary

+

Brief summary

${summary} <#if author??>

Author

${author} -

Description

+

Detailed description

${description} <#-- Create the argument summary --> @@ -57,10 +74,10 @@ - <#-- Create references to related capabilities if appropriate --> + <#-- Create references to additional capabilities if appropriate --> <#if extradocs?size != 0>
-

Related capabilities

+

Additional capabilities

The arguments described in the entries below can be supplied to this tool to modify its behavior. For example, the -L argument directs the GATK engine restricts processing to specific genomic intervals. This capability is available to all GATK walkers. @@ -70,6 +87,14 @@ + + <#-- This class is related to other documented classes via sub/super relationships --> + <#if relatedDocs?size != 0> +

Related capabilities

+ <@relatedByType name="Superclasses" type="superclass"/> + <@relatedByType name="Subclasses" type="subclass"/> +
+ <#-- List all of the --> <#if arguments.all?size != 0> From 5e0fe2d0f976aceb1aac53ea0d8db09bccd7633c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 15:42:39 -0400 Subject: [PATCH 27/41] Support for style.css via refactored common.html included in all files --- .../sting/utils/help/GATKDoclet.java | 14 +++++++++++--- settings/helpTemplates/generic.index.template.html | 8 ++++---- settings/helpTemplates/generic.template.html | 13 ++++++------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index aa5514dae..3b581e77b 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -30,8 +30,10 @@ import freemarker.template.Configuration; import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; +import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; +import sun.misc.IOUtils; import java.io.*; import java.util.*; @@ -40,6 +42,8 @@ import java.util.*; * */ public class GATKDoclet extends ResourceBundleExtractorDoclet { + final protected static File SETTINGS_DIR = new File("settings/helpTemplates"); + final protected static File DESTINATION_DIR = new File("testdoc"); final protected static Logger logger = Logger.getLogger(GATKDoclet.class); public static class DocWorkUnit implements Comparable { @@ -128,12 +132,16 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { this.rootDoc = rootDoc; try { + // basic setup + DESTINATION_DIR.mkdirs(); + FileUtils.copyFile(new File(SETTINGS_DIR + "/style.css"), new File(DESTINATION_DIR + "/style.css")); + /* ------------------------------------------------------------------- */ /* You should do this ONLY ONCE in the whole application life-cycle: */ Configuration cfg = new Configuration(); // Specify the data source where the template files come from. - cfg.setDirectoryForTemplateLoading(new File("settings/helpTemplates/")); + cfg.setDirectoryForTemplateLoading(SETTINGS_DIR); // Specify how templates will see the data-model. This is an advanced topic... cfg.setObjectWrapper(new DefaultObjectWrapper()); @@ -200,7 +208,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Template temp = cfg.getTemplate("generic.index.template.html"); /* Merge data-model with template */ - Writer out = new OutputStreamWriter(new FileOutputStream(new File("testdoc/index.html"))); + Writer out = new OutputStreamWriter(new FileOutputStream(new File(DESTINATION_DIR + "/index.html"))); try { temp.process(groupIndexData(indexData), out); out.flush(); @@ -259,7 +267,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Template temp = cfg.getTemplate(unit.handler.getTemplateName(unit.classDoc)); // Merge data-model with template - File outputPath = new File("testdoc/" + unit.filename); + File outputPath = new File(DESTINATION_DIR + "/" + unit.filename); try { Writer out = new OutputStreamWriter(new FileOutputStream(outputPath)); temp.process(unit.forTemplate, out); diff --git a/settings/helpTemplates/generic.index.template.html b/settings/helpTemplates/generic.index.template.html index b494dc582..a6ea3da86 100644 --- a/settings/helpTemplates/generic.index.template.html +++ b/settings/helpTemplates/generic.index.template.html @@ -1,5 +1,7 @@ +<#include "common.html"/> + <#macro emitGroup group> - +

${group.name}

${group.summary} @@ -20,9 +22,7 @@ - - GATK documentation index - +<@makeHeader title="GATK documentation index"/>

GATK documentation index

<#list groups as group> diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index 6540ba0e3..d029cf8a5 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -1,3 +1,5 @@ +<#include "common.html"/> + <#macro argumentlist name myargs> <#if myargs?size != 0> @@ -39,12 +41,9 @@ - - + - - ${name} documentation - +<@makeHeader title="${name} documentation"/>

${name}

Brief summary

@@ -60,7 +59,7 @@ <#if arguments.all?size != 0>

Feature specific arguments

-
${name}
+
@@ -90,10 +89,10 @@ <#-- This class is related to other documented classes via sub/super relationships --> <#if relatedDocs?size != 0> +

Related capabilities

<@relatedByType name="Superclasses" type="superclass"/> <@relatedByType name="Subclasses" type="subclass"/> -
<#-- List all of the --> From b8db0510e0eb1ed2ae4970e79b698dc1d372fc9b Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 15:43:23 -0400 Subject: [PATCH 28/41] Sytle sheet and common functions --- settings/helpTemplates/common.html | 6 +++ settings/helpTemplates/style.css | 80 ++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 settings/helpTemplates/common.html create mode 100644 settings/helpTemplates/style.css diff --git a/settings/helpTemplates/common.html b/settings/helpTemplates/common.html new file mode 100644 index 000000000..615210e07 --- /dev/null +++ b/settings/helpTemplates/common.html @@ -0,0 +1,6 @@ +<#macro makeHeader title> + + ${title} + + + diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css new file mode 100644 index 000000000..e513bb527 --- /dev/null +++ b/settings/helpTemplates/style.css @@ -0,0 +1,80 @@ +body +{ + background-color: #ffffff; + color: #202020; +} + +body, p, ul, ol, dl +{ + font-family: Corbel, Verdana, "Lucida Grande", "Lucida Sans Unicode", Sans-Serif; +} + +p, ul, ol, dl, dt, dd, td +{ + font-size: 12pt; +} + +dt +{ + padding-bottom: 6pt; +} + +/* +a +{ + color: #546188; + text-decoration: none; +} + +a:visited +{ + color: #546188; +} +*/ + +h1, h2, h3 +{ + font-family: Corbel, Arial, Helvetica, Sans-Serif; + font-weight: lighter; + text-align: left; + color: #bac8da; +} + +h1 +{ + font-size: 32pt; + letter-spacing: -2px; +} + +h3 +{ + font-size: 16pt; + font-weight: normal; +} + +#indexheader +{ + position: absolute; + left: 50pt; + top: 100pt; + font-size: 38pt; +} + +#indexbody +{ + position: absolute; + left: 50pt; + top: 180pt; +} + +table#indextable tr td +{ + margin-right: 1em; +} + +tr#indextableheader td +{ + font-family: Corbel, Arial, Helvetica, Sans-Serif; + font-size: 14pt; + font-weight: lighter; +} From c6af4efcdc7f8249bbf87b516fa7025ce268f74c Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 16:10:17 -0400 Subject: [PATCH 29/41] Implemented see also and version header --- build.xml | 2 +- .../sting/utils/help/GATKDoclet.java | 29 ++++++++++++------- .../help/GenericDocumentationHandler.java | 2 ++ settings/helpTemplates/common.html | 9 ++++++ .../helpTemplates/generic.index.template.html | 3 ++ settings/helpTemplates/generic.template.html | 5 +++- settings/helpTemplates/style.css | 7 ++++- 7 files changed, 44 insertions(+), 13 deletions(-) diff --git a/build.xml b/build.xml index ed7802b1d..fc495f7cc 100644 --- a/build.xml +++ b/build.xml @@ -479,7 +479,7 @@ docletpathref="doclet.classpath" classpathref="external.dependencies" classpath="${java.classes}" - additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -out ${basedir}/${resource.path} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> + additionalparam="-private -build-timestamp "${build.timestamp}" -absolute-version ${build.version} -quiet -J-Xdebug -J-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"> diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 3b581e77b..c1dd843ac 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -48,18 +48,21 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { public static class DocWorkUnit implements Comparable { // known at the start - String name, filename, group; - DocumentedGATKFeatureHandler handler; - ClassDoc classDoc; - Class clazz; - DocumentedGATKFeature annotation; + final String name, filename, group; + final DocumentedGATKFeatureHandler handler; + final ClassDoc classDoc; + final Class clazz; + final DocumentedGATKFeature annotation; + final String buildTimestamp, absoluteVersion; // set by the handler String summary; Map forTemplate; - public DocWorkUnit(DocumentedGATKFeature annotation, String name, String filename, String group, - DocumentedGATKFeatureHandler handler, ClassDoc classDoc, Class clazz) { + public DocWorkUnit(String name, String filename, String group, + DocumentedGATKFeature annotation, DocumentedGATKFeatureHandler handler, + ClassDoc classDoc, Class clazz, + String buildTimestamp, String absoluteVersion) { this.annotation = annotation; this.name = name; this.filename = filename; @@ -67,6 +70,8 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { this.handler = handler; this.classDoc = classDoc; this.clazz = clazz; + this.buildTimestamp = buildTimestamp; + this.absoluteVersion = absoluteVersion; } public void setHandlerContent(String summary, Map forTemplate) { @@ -116,9 +121,10 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { DocumentedGATKFeatureHandler handler = createHandler(doc, feature); if ( handler != null && handler.shouldBeProcessed(doc) ) { String filename = handler.getDestinationFilename(doc); - DocWorkUnit unit = new DocWorkUnit(feature, - doc.name(), filename, feature.groupName(), - handler, doc, clazz ); + DocWorkUnit unit = new DocWorkUnit(doc.name(), + filename, feature.groupName(), + feature, handler, doc, clazz, + buildTimestamp, absoluteVersion); m.add(unit); } } @@ -130,6 +136,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { protected void processDocs(RootDoc rootDoc, PrintStream ignore) { // setup the global access to the root this.rootDoc = rootDoc; + super.loadData(rootDoc, false); try { // basic setup @@ -239,6 +246,8 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { root.put("data", data); root.put("groups", groups); + root.put("timestamp", buildTimestamp); + root.put("version", absoluteVersion); return root; } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index cbe526793..777a498c0 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -88,6 +88,8 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { summaryBuilder.append(tag.text()); root.put("summary", summaryBuilder.toString()); root.put("description", classdoc.commentText()); + root.put("timestamp", toProcess.buildTimestamp); + root.put("version", toProcess.absoluteVersion); for(Tag tag: classdoc.tags()) { root.put(tag.name(), tag.text()); diff --git a/settings/helpTemplates/common.html b/settings/helpTemplates/common.html index 615210e07..ebc060d0a 100644 --- a/settings/helpTemplates/common.html +++ b/settings/helpTemplates/common.html @@ -4,3 +4,12 @@ + +<#macro headerInfo> +

See also Main index | GATK wiki | GATK support forum

+

GATK version ${version} built at ${timestamp}.

+ + +<#macro footerInfo> + + diff --git a/settings/helpTemplates/generic.index.template.html b/settings/helpTemplates/generic.index.template.html index a6ea3da86..56eae97c2 100644 --- a/settings/helpTemplates/generic.index.template.html +++ b/settings/helpTemplates/generic.index.template.html @@ -25,9 +25,12 @@ <@makeHeader title="GATK documentation index"/>

GATK documentation index

+ <@headerInfo /> <#list groups as group> <@emitGroup group=group/> + + <@footerInfo /> diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index d029cf8a5..11823687d 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -45,7 +45,8 @@ <@makeHeader title="${name} documentation"/> -

${name}

+

${name}

+ <@headerInfo />

Brief summary

${summary} <#if author??> @@ -104,5 +105,7 @@ <@argumentDetails arg=arg/> + + <@footerInfo /> diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css index e513bb527..1b3067044 100644 --- a/settings/helpTemplates/style.css +++ b/settings/helpTemplates/style.css @@ -14,6 +14,11 @@ p, ul, ol, dl, dt, dd, td font-size: 12pt; } +p.version, p.see-also +{ + font-size: 6pt; +} + dt { padding-bottom: 6pt; @@ -35,7 +40,7 @@ a:visited h1, h2, h3 { font-family: Corbel, Arial, Helvetica, Sans-Serif; - font-weight: lighter; + font-weight: bold; text-align: left; color: #bac8da; } From 793e7d3d1dd9df622fa09cc174312525da6febfb Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 16:36:25 -0400 Subject: [PATCH 30/41] Improved header and argument details Argument detail structure cleaned up. Only relevant pieces of information are shown now, and in a cleaner layout. Misc. cleanup in the code. --- .../walkers/diffengine/DiffObjectsWalker.java | 4 +++ .../help/GenericDocumentationHandler.java | 25 +++++++++++-------- settings/helpTemplates/generic.template.html | 10 +++----- settings/helpTemplates/style.css | 2 +- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index 98b1eb13d..f80e85708 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -186,6 +186,10 @@ public class DiffObjectsWalker extends RodWalker { @Argument(fullName="testEnum", doc="X", required=false) TestEnum testEnum = TestEnum.ONE; + @Deprecated + @Argument(fullName="testDepreciates", doc="Y", required=false) + int dontUseMe = 1; + public enum TestEnum { ONE, TWO }; final DiffEngine diffEngine = new DiffEngine(); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 777a498c0..6d982daf0 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -75,7 +75,6 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { addArgumentBindings(root); addRelatedBindings(root); - //System.out.printf("Root is %s%n", root); toProcess.setHandlerContent((String)root.get("summary"), root); } @@ -110,7 +109,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(HelpUtils.getClassForDoc(classdoc)) ) { ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); - GATKDoc doc = docForArgument(fieldDoc, argDef); // todo -- why can you have multiple ones? + GATKDoc doc = docForArgument(fieldDoc, argumentSource, argDef); // todo -- why can you have multiple ones? String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; @@ -216,7 +215,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { return null; } - protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentDefinition def) { + protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentSource source, ArgumentDefinition def) { final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); @@ -228,17 +227,23 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { if ( def.doc != null ) doc.setSummary(def.doc); List attributes = new ArrayList(); - attributes.add(def.ioType.annotationClass.getSimpleName()); + // this one below is just too much. + //attributes.add(def.ioType.annotationClass.getSimpleName()); if ( def.required ) attributes.add("required"); - if ( def.isFlag ) attributes.add("flag"); + // flag is just boolean, not interesting + //if ( def.isFlag ) attributes.add("flag"); if ( def.isHidden ) attributes.add("hidden"); - doc.addTag("attributes", Utils.join(",", attributes)); + if ( source.isDeprecated() ) attributes.add("depreciated"); + if ( attributes.size() > 0 ) + doc.addTag("attributes", Utils.join(", ", attributes)); - // todo -- need depreciated value + if ( def.validOptions != null ) { + //source.field.getType().isEnum(); + // todo -- what's the best way to link to these docs? Maybe a separate section on enums? + doc.addTag("options", Utils.join(", ", def.validOptions)); + } - doc.addTag("options", def.validOptions == null ? GATKDoc.NA_STRING : Utils.join(",", def.validOptions)); - - doc.setFulltext(fieldDoc.commentText()); + doc.setFulltext(fieldDoc.commentText().equals("") ? GATKDoc.NA_STRING : fieldDoc.commentText()); return doc; } diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index 11823687d..108d6ad5c 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -18,12 +18,10 @@ <#macro argumentDetails arg> -

${arg.name} / ${arg.synonyms}

- Summary: ${arg.summary}
- Attributes: ${arg.attributes}
- Type: ${arg.type}
- Options: ${arg.options}
- Details: ${arg.fulltext}
+

${arg.name} / ${arg.synonyms} + (<#if arg.attributes??>${arg.attributes} ${arg.type})

+ ${arg.summary}. ${arg.fulltext}
+ <#if arg.options??>TODO: document enum in line here. Possible values ${arg.options} <#macro relatedByType name type> diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css index 1b3067044..85ecf8a02 100644 --- a/settings/helpTemplates/style.css +++ b/settings/helpTemplates/style.css @@ -16,7 +16,7 @@ p, ul, ol, dl, dt, dd, td p.version, p.see-also { - font-size: 6pt; + font-size: 8pt; } dt From c620d96c96e727d5d58bed6c1ace1bfa87ef61ee Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 17:22:14 -0400 Subject: [PATCH 31/41] Inline enum documentation is working --- .../walkers/diffengine/DiffObjectsWalker.java | 7 +- .../help/DocumentedGATKFeatureHandler.java | 2 +- .../sting/utils/help/GATKDoc.java | 123 ------------------ .../sting/utils/help/GATKDoclet.java | 6 +- .../help/GenericDocumentationHandler.java | 57 +++++--- settings/helpTemplates/generic.template.html | 14 +- settings/helpTemplates/style.css | 23 +++- 7 files changed, 82 insertions(+), 150 deletions(-) delete mode 100644 public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index f80e85708..c660bda36 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -190,7 +190,12 @@ public class DiffObjectsWalker extends RodWalker { @Argument(fullName="testDepreciates", doc="Y", required=false) int dontUseMe = 1; - public enum TestEnum { ONE, TWO }; + public enum TestEnum { + /** Docs for ONE */ + ONE, + /** Docs for TWO */ + TWO + }; final DiffEngine diffEngine = new DiffEngine(); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index f25a02bb5..198b09ebc 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -52,5 +52,5 @@ public abstract class DocumentedGATKFeatureHandler { } public abstract String getTemplateName(ClassDoc doc) throws IOException; - public abstract void processOne(GATKDoclet.DocWorkUnit toProcess, Set all); + public abstract void processOne(RootDoc rootDoc, GATKDoclet.DocWorkUnit toProcess, Set all); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java deleted file mode 100644 index 0c575523e..000000000 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoc.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2011, The Broad Institute - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the - * Software is furnished to do so, subject to the following - * conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES - * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR - * OTHER DEALINGS IN THE SOFTWARE. - */ - -package org.broadinstitute.sting.utils.help; - -import org.broadinstitute.sting.utils.Utils; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Created by IntelliJ IDEA. - * User: depristo - * Date: 7/21/11 - * Time: 8:51 AM - * - * Common documentation information about an GATK capability - */ -public class GATKDoc { - final DocType type; - final String name; - List synonyms; - String summary, fulltext; - final Map tags; - - public final static String NA_STRING = "None provided"; - - public enum DocType { - WALKER ("Walker"), - WALKER_ARG ("Walker argument"), - READ_FILTER ("Read filter"), - ENGINE_FEATURE ("Engine feature"); - - private final String name; - DocType(String name) { - this.name = name; - } - - }; - - public GATKDoc(DocType type, String name) { - this(type, name, new ArrayList(), NA_STRING, NA_STRING, new HashMap()); - } - - public GATKDoc(DocType type, String name, List synonyms, String summary, String fulltext, Map tags) { - this.type = type; - this.name = name; - this.synonyms = synonyms; - this.summary = summary; - this.fulltext = fulltext; - this.tags = tags; - } - - public Map toDataModel() { - Map model = new HashMap(); - model.put("type", type.name); - model.put("name", name); - model.put("synonyms", Utils.join(",", synonyms)); - model.put("summary", summary); - model.put("fulltext", fulltext); - model.putAll(tags); - return model; - } - - public DocType getType() { - return type; - } - - public String getName() { - return name; - } - - public List getSynonyms() { - return synonyms; - } - - public void addSynonym(String synonyms) { - this.synonyms.add(synonyms); - } - - public String getSummary() { - return summary; - } - - public void setSummary(String summary) { - this.summary = summary; - } - - public String getFulltext() { - return fulltext; - } - - public void setFulltext(String fulltext) { - this.fulltext = fulltext; - } - - public void addTag(String key, String value) { - this.tags.put(key, value); - } -} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index c1dd843ac..db7d6651d 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -210,6 +210,10 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } + public static ClassDoc getClassDocForClass(RootDoc rootDoc, Class clazz) { + return rootDoc.classNamed(clazz.getName()); + } + private void processIndex(Configuration cfg, List indexData) throws IOException { /* Get or create a template */ Template temp = cfg.getTemplate("generic.index.template.html"); @@ -270,7 +274,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { throws IOException { System.out.printf("Processing documentation for class %s%n", unit.classDoc); - unit.handler.processOne(unit, all); + unit.handler.processOne(rootDoc, unit, all); // Get or create a template Template temp = cfg.getTemplate(unit.handler.getTemplateName(unit.classDoc)); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 6d982daf0..ff54a115c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -24,8 +24,10 @@ package org.broadinstitute.sting.utils.help; +import com.google.java.contract.Requires; import com.sun.javadoc.ClassDoc; import com.sun.javadoc.FieldDoc; +import com.sun.javadoc.RootDoc; import com.sun.javadoc.Tag; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.CommandLineGATK; @@ -44,6 +46,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { GATKDoclet.DocWorkUnit toProcess; ClassDoc classdoc; Set all; + RootDoc rootDoc; @Override public boolean shouldBeProcessed(ClassDoc doc) { @@ -63,7 +66,8 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } @Override - public void processOne(GATKDoclet.DocWorkUnit toProcessArg, Set allArg) { + public void processOne(RootDoc rootDoc, GATKDoclet.DocWorkUnit toProcessArg, Set allArg) { + this.rootDoc = rootDoc; this.toProcess = toProcessArg; this.all = allArg; this.classdoc = toProcess.classDoc; @@ -109,13 +113,13 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { for ( ArgumentSource argumentSource : parsingEngine.extractArgumentSources(HelpUtils.getClassForDoc(classdoc)) ) { ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); - GATKDoc doc = docForArgument(fieldDoc, argumentSource, argDef); // todo -- why can you have multiple ones? + Map argBindings = docForArgument(fieldDoc, argumentSource, argDef); // todo -- why can you have multiple ones? String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(doc.toDataModel()); - args.get("all").add(doc.toDataModel()); + args.get(kind).add(argBindings); + args.get("all").add(argBindings); System.out.printf("Processing %s%n", argumentSource); } } catch ( ClassNotFoundException e ) { @@ -215,16 +219,19 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { return null; } - protected GATKDoc docForArgument(FieldDoc fieldDoc, ArgumentSource source, ArgumentDefinition def) { - final String name = def.fullName != null ? "--" + def.fullName : "-" + def.shortName; - GATKDoc doc = new GATKDoc(GATKDoc.DocType.WALKER_ARG, name); + protected Map docForArgument(FieldDoc fieldDoc, ArgumentSource source, ArgumentDefinition def) { + Map root = new HashMap(); + root.put("name", def.fullName != null ? "--" + def.fullName : "-" + def.shortName); if ( def.fullName != null && def.shortName != null) - doc.addSynonym("-" + def.shortName); + root.put("synonyms", def.fullName != null ? "--" + def.fullName : "-" + def.shortName); - doc.addTag("required", def.required ? "yes" : "no"); - doc.addTag("type", def.argumentType.getSimpleName()); - if ( def.doc != null ) doc.setSummary(def.doc); + root.put("required", def.required ? "yes" : "no"); + root.put("type", def.argumentType.getSimpleName()); + + // summary and fulltext + root.put("summary", def.doc != null ? def.doc : ""); + root.put("fulltext", fieldDoc.commentText()); List attributes = new ArrayList(); // this one below is just too much. @@ -235,16 +242,30 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { if ( def.isHidden ) attributes.add("hidden"); if ( source.isDeprecated() ) attributes.add("depreciated"); if ( attributes.size() > 0 ) - doc.addTag("attributes", Utils.join(", ", attributes)); + root.put("attributes", Utils.join(", ", attributes)); if ( def.validOptions != null ) { - //source.field.getType().isEnum(); - // todo -- what's the best way to link to these docs? Maybe a separate section on enums? - doc.addTag("options", Utils.join(", ", def.validOptions)); + root.put("options", docForEnumArgument(source.field.getType())); } - doc.setFulltext(fieldDoc.commentText().equals("") ? GATKDoc.NA_STRING : fieldDoc.commentText()); - - return doc; + return root; } + + @Requires("enumClass.isEnum()") + private List> docForEnumArgument(Class enumClass) { + ClassDoc doc = GATKDoclet.getClassDocForClass(rootDoc, enumClass); + if ( doc == null ) // || ! doc.isEnum() ) + throw new RuntimeException("Tried to get docs for enum " + enumClass + " but got instead: " + doc); + + List> bindings = new ArrayList>(); + for (final FieldDoc field : doc.fields(false) ) { + bindings.add( + new HashMap(){{ + put("name", field.name()); + put("summary", field.commentText());}}); + } + + return bindings; + } + } diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index 108d6ad5c..b8a392686 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -6,7 +6,7 @@ <#list myargs as arg> - + @@ -18,10 +18,18 @@ <#macro argumentDetails arg> -

${arg.name} / ${arg.synonyms} +

${arg.name}<#if arg.synonyms??> / ${arg.synonyms} (<#if arg.attributes??>${arg.attributes} ${arg.type})

${arg.summary}. ${arg.fulltext}
- <#if arg.options??>TODO: document enum in line here. Possible values ${arg.options} + <#if arg.options??> +

Argument is an enumerated type with the following possible values:

+
+ <#list arg.options as option> +
${option.name} +
${option.summary} + +
+ <#macro relatedByType name type> diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css index 85ecf8a02..70aadcf79 100644 --- a/settings/helpTemplates/style.css +++ b/settings/helpTemplates/style.css @@ -19,9 +19,26 @@ p.version, p.see-also font-size: 8pt; } -dt -{ - padding-bottom: 6pt; +dl.enum +{ + margin: 0em 0; + padding: 0; +} + +.enum dt +{ + position: relative; + left: 0; + top: 1.1em; + width: 5em; + font-weight: bold; +} + +.enum dd +{ + border-left: 1px solid #000; + margin: 0 0 0 6em; + padding: 0 0 .5em .5em; } /* From 3c34e9fa659624498db3872f560e94ac96ecaa82 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 17:45:58 -0400 Subject: [PATCH 32/41] Cleanup emuns and tables --- .../sting/utils/help/GATKDoclet.java | 2 +- .../helpTemplates/generic.index.template.html | 2 +- settings/helpTemplates/generic.template.html | 10 ++-- settings/helpTemplates/style.css | 52 ++++++++++++------- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index db7d6651d..38a3f4e18 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -43,7 +43,7 @@ import java.util.*; */ public class GATKDoclet extends ResourceBundleExtractorDoclet { final protected static File SETTINGS_DIR = new File("settings/helpTemplates"); - final protected static File DESTINATION_DIR = new File("testdoc"); + final protected static File DESTINATION_DIR = new File("gatkdocs"); final protected static Logger logger = Logger.getLogger(GATKDoclet.class); public static class DocWorkUnit implements Comparable { diff --git a/settings/helpTemplates/generic.index.template.html b/settings/helpTemplates/generic.index.template.html index 56eae97c2..b9abd7909 100644 --- a/settings/helpTemplates/generic.index.template.html +++ b/settings/helpTemplates/generic.index.template.html @@ -1,7 +1,7 @@ <#include "common.html"/> <#macro emitGroup group> -
Name Synonyms
${arg.name}${arg.synonyms}${arg.synonyms!""} ${arg.type} ${arg.summary}
+

${group.name}

${group.summary} diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index b8a392686..cea7d0631 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -2,7 +2,7 @@ <#macro argumentlist name myargs> <#if myargs?size != 0> -

+ <#list myargs as arg> @@ -22,7 +22,7 @@ (<#if arg.attributes??>${arg.attributes} ${arg.type}) ${arg.summary}. ${arg.fulltext}
<#if arg.options??> -

Argument is an enumerated type with the following possible values:

+

The ${arg.name} argument can have one of the following values:

<#list arg.options as option>
${option.name} @@ -66,17 +66,21 @@ <#if arguments.all?size != 0>

Feature specific arguments

-
${name}
${name}
${arg.name}
+
+ + + <@argumentlist name="Required" myargs=arguments.required/> <@argumentlist name="Optional" myargs=arguments.optional/> <@argumentlist name="Hidden" myargs=arguments.hidden/> <@argumentlist name="Depreciated" myargs=arguments.depreciated/> +
Name Synonyms Type Summary
diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css index 70aadcf79..18c9fe521 100644 --- a/settings/helpTemplates/style.css +++ b/settings/helpTemplates/style.css @@ -74,29 +74,41 @@ h3 font-weight: normal; } -#indexheader -{ - position: absolute; - left: 50pt; - top: 100pt; - font-size: 38pt; -} - -#indexbody -{ - position: absolute; - left: 50pt; - top: 180pt; -} - -table#indextable tr td +table.indextable tr td { margin-right: 1em; } -tr#indextableheader td +/* + Blue Dream + Written by Teylor Feliz http://www.admixweb.com +*/ + +#hor-minimalist-b { - font-family: Corbel, Arial, Helvetica, Sans-Serif; - font-size: 14pt; - font-weight: lighter; + font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif; + font-size: 12px; + background: #fff; + margin: 5px; + width: 100%; + border-collapse: collapse; + text-align: left; } +#hor-minimalist-b th +{ + font-size: 14px; + font-weight: normal; + color: #039; + padding: 10px 8px; + border-bottom: 2px solid #6678b1; +} +#hor-minimalist-b td +{ + border-bottom: 1px solid #ccc; + color: #669; + padding: 6px 8px; +} +#hor-minimalist-b tbody tr:hover td +{ + color: #009; +} \ No newline at end of file From 83996f7951920d161de4143764323f06b235685f Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 18:14:21 -0400 Subject: [PATCH 33/41] Enumerated types are working. --- .../sting/gatk/phonehome/GATKRunReport.java | 4 ++ .../help/GenericDocumentationHandler.java | 6 +- settings/helpTemplates/generic.template.html | 6 +- settings/helpTemplates/style.css | 70 +++++++------------ 4 files changed, 35 insertions(+), 51 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index 69c0b3e0a..acee1a6a3 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -154,9 +154,13 @@ public class GATKRunReport { private long nReads; public enum PhoneHomeOption { + /** Disable phone home */ NO_ET, + /** Standard option. Writes to local repository if it can be found, or S3 otherwise */ STANDARD, + /** Force output to STDOUT. For debugging only */ STDOUT, + /** Force output to S3. For debugging only */ AWS_S3 // todo -- remove me -- really just for testing purposes } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index ff54a115c..5fd6f27d9 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -221,10 +221,10 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { protected Map docForArgument(FieldDoc fieldDoc, ArgumentSource source, ArgumentDefinition def) { Map root = new HashMap(); - root.put("name", def.fullName != null ? "--" + def.fullName : "-" + def.shortName); + root.put("name", def.shortName != null ? "-" + def.shortName : "--" + def.fullName ); - if ( def.fullName != null && def.shortName != null) - root.put("synonyms", def.fullName != null ? "--" + def.fullName : "-" + def.shortName); + if ( def.shortName != null && def.fullName != null ) + root.put("synonyms", "--" + def.fullName); root.put("required", def.required ? "yes" : "no"); root.put("type", def.argumentType.getSimpleName()); diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index cea7d0631..d2468214c 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -2,11 +2,10 @@ <#macro argumentlist name myargs> <#if myargs?size != 0> - ${name} + ${name} <#list myargs as arg> ${arg.name} - ${arg.synonyms!""} ${arg.type} ${arg.summary} @@ -22,7 +21,7 @@ (<#if arg.attributes??>${arg.attributes} ${arg.type}) ${arg.summary}. ${arg.fulltext}
<#if arg.options??> -

The ${arg.name} argument can have one of the following values:

+

The ${arg.name} argument is an enumerated type (${arg.type}), which can have one of the following values:

<#list arg.options as option>
${option.name} @@ -70,7 +69,6 @@ Name - Synonyms Type Summary diff --git a/settings/helpTemplates/style.css b/settings/helpTemplates/style.css index 18c9fe521..79f409f55 100644 --- a/settings/helpTemplates/style.css +++ b/settings/helpTemplates/style.css @@ -19,47 +19,12 @@ p.version, p.see-also font-size: 8pt; } -dl.enum -{ - margin: 0em 0; - padding: 0; -} - -.enum dt -{ - position: relative; - left: 0; - top: 1.1em; - width: 5em; - font-weight: bold; -} - -.enum dd -{ - border-left: 1px solid #000; - margin: 0 0 0 6em; - padding: 0 0 .5em .5em; -} - -/* -a -{ - color: #546188; - text-decoration: none; -} - -a:visited -{ - color: #546188; -} -*/ - h1, h2, h3 { font-family: Corbel, Arial, Helvetica, Sans-Serif; font-weight: bold; text-align: left; - color: #bac8da; + color: #669; } h1 @@ -74,16 +39,27 @@ h3 font-weight: normal; } -table.indextable tr td -{ - margin-right: 1em; -} - -/* - Blue Dream - Written by Teylor Feliz http://www.admixweb.com +/* + * enum DT layout */ +dl { + border: 1px solid #ccc; +} + +dt { + font-weight: bold; + text-decoration: underline; +} + +dd { + margin: 0; + padding: 0 0 0.5em 0; +} + +/* + * clean table layouts +*/ #hor-minimalist-b { font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif; @@ -111,4 +87,10 @@ table.indextable tr td #hor-minimalist-b tbody tr:hover td { color: #009; +} + +th#row-divider +{ + font-weight: bolder; + font-size: larger; } \ No newline at end of file From 7b84347047afec7ce71f94d7d70adfa55bee483d Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 18:19:40 -0400 Subject: [PATCH 34/41] Main index is sorted by the template now. --- settings/helpTemplates/generic.index.template.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/helpTemplates/generic.index.template.html b/settings/helpTemplates/generic.index.template.html index b9abd7909..6c9e9f4e8 100644 --- a/settings/helpTemplates/generic.index.template.html +++ b/settings/helpTemplates/generic.index.template.html @@ -26,7 +26,7 @@

GATK documentation index

<@headerInfo /> - <#list groups as group> + <#list groups?sort_by("name") as group> <@emitGroup group=group/> From 9f06f6c4930cb317f70c7902cf099366cca5856e Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 20:00:04 -0400 Subject: [PATCH 36/41] Split GATKDoclet from ResourceBundleDoclet. Refactored GaTKDocWorkUnit --- .../help/DocumentedGATKFeatureHandler.java | 3 +- .../sting/utils/help/GATKDocWorkUnit.java | 84 ++++++++++++++++ .../sting/utils/help/GATKDoclet.java | 97 +++++++------------ .../help/GenericDocumentationHandler.java | 11 +-- 4 files changed, 123 insertions(+), 72 deletions(-) create mode 100644 public/java/src/org/broadinstitute/sting/utils/help/GATKDocWorkUnit.java diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index 198b09ebc..2227f9aba 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -28,7 +28,6 @@ import com.sun.javadoc.ClassDoc; import com.sun.javadoc.RootDoc; import java.io.*; -import java.util.Map; import java.util.Set; /** @@ -52,5 +51,5 @@ public abstract class DocumentedGATKFeatureHandler { } public abstract String getTemplateName(ClassDoc doc) throws IOException; - public abstract void processOne(RootDoc rootDoc, GATKDoclet.DocWorkUnit toProcess, Set all); + public abstract void processOne(RootDoc rootDoc, GATKDocWorkUnit toProcess, Set all); } diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDocWorkUnit.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDocWorkUnit.java new file mode 100644 index 000000000..65c6624d5 --- /dev/null +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDocWorkUnit.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.utils.help; + +import com.sun.javadoc.ClassDoc; + +import java.util.HashMap; +import java.util.Map; + +/** +* Created by IntelliJ IDEA. +* User: depristo +* Date: 7/24/11 +* Time: 7:59 PM +* To change this template use File | Settings | File Templates. +*/ +public class GATKDocWorkUnit implements Comparable { + // known at the start + final String name, filename, group; + final DocumentedGATKFeatureHandler handler; + final ClassDoc classDoc; + final Class clazz; + final DocumentedGATKFeature annotation; + final String buildTimestamp, absoluteVersion; + + // set by the handler + String summary; + Map forTemplate; + + public GATKDocWorkUnit(String name, String filename, String group, + DocumentedGATKFeature annotation, DocumentedGATKFeatureHandler handler, + ClassDoc classDoc, Class clazz, + String buildTimestamp, String absoluteVersion) { + this.annotation = annotation; + this.name = name; + this.filename = filename; + this.group = group; + this.handler = handler; + this.classDoc = classDoc; + this.clazz = clazz; + this.buildTimestamp = buildTimestamp; + this.absoluteVersion = absoluteVersion; + } + + public void setHandlerContent(String summary, Map forTemplate) { + this.summary = summary; + this.forTemplate = forTemplate; + } + + public Map toMap() { + Map data = new HashMap(); + data.put("name", name); + data.put("summary", summary); + data.put("filename", filename); + data.put("group", group); + return data; + } + + public int compareTo(GATKDocWorkUnit other) { + return this.name.compareTo(other.name); + } +} diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 38a3f4e18..ff20afbfa 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -33,7 +33,6 @@ import freemarker.template.TemplateException; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; -import sun.misc.IOUtils; import java.io.*; import java.util.*; @@ -41,57 +40,11 @@ import java.util.*; /** * */ -public class GATKDoclet extends ResourceBundleExtractorDoclet { +public class GATKDoclet { final protected static File SETTINGS_DIR = new File("settings/helpTemplates"); final protected static File DESTINATION_DIR = new File("gatkdocs"); final protected static Logger logger = Logger.getLogger(GATKDoclet.class); - - public static class DocWorkUnit implements Comparable { - // known at the start - final String name, filename, group; - final DocumentedGATKFeatureHandler handler; - final ClassDoc classDoc; - final Class clazz; - final DocumentedGATKFeature annotation; - final String buildTimestamp, absoluteVersion; - - // set by the handler - String summary; - Map forTemplate; - - public DocWorkUnit(String name, String filename, String group, - DocumentedGATKFeature annotation, DocumentedGATKFeatureHandler handler, - ClassDoc classDoc, Class clazz, - String buildTimestamp, String absoluteVersion) { - this.annotation = annotation; - this.name = name; - this.filename = filename; - this.group = group; - this.handler = handler; - this.classDoc = classDoc; - this.clazz = clazz; - this.buildTimestamp = buildTimestamp; - this.absoluteVersion = absoluteVersion; - } - - public void setHandlerContent(String summary, Map forTemplate) { - this.summary = summary; - this.forTemplate = forTemplate; - } - - public Map toMap() { - Map data = new HashMap(); - data.put("name", name); - data.put("summary", summary); - data.put("filename", filename); - data.put("group", group); - return data; - } - - public int compareTo(DocWorkUnit other) { - return this.name.compareTo(other.name); - } - } + protected static String buildTimestamp = null, absoluteVersion = null; RootDoc rootDoc; @@ -102,17 +55,33 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { * @throws java.io.IOException if output can't be written. */ public static boolean start(RootDoc rootDoc) throws IOException { + // load arguments + for(String[] options: rootDoc.options()) { + if(options[0].equals("-build-timestamp")) + buildTimestamp = options[1]; + if (options[0].equals("-absolute-version")) + absoluteVersion = options[1]; + } + GATKDoclet doclet = new GATKDoclet(); - doclet.processDocs(rootDoc, null); + doclet.processDocs(rootDoc); return true; } + /** + * Validate the given options against options supported by this doclet. + * @param option Option to validate. + * @return Number of potential parameters; 0 if not supported. + */ public static int optionLength(String option) { - return ResourceBundleExtractorDoclet.optionLength(option); + if(option.equals("-build-timestamp") || option.equals("-absolute-version") ) { + return 2; + } + return 0; } - public Set workUnits() { - TreeSet m = new TreeSet(); + public Set workUnits() { + TreeSet m = new TreeSet(); for ( ClassDoc doc : rootDoc.classes() ) { System.out.printf("Considering %s%n", doc); @@ -121,7 +90,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { DocumentedGATKFeatureHandler handler = createHandler(doc, feature); if ( handler != null && handler.shouldBeProcessed(doc) ) { String filename = handler.getDestinationFilename(doc); - DocWorkUnit unit = new DocWorkUnit(doc.name(), + GATKDocWorkUnit unit = new GATKDocWorkUnit(doc.name(), filename, feature.groupName(), feature, handler, doc, clazz, buildTimestamp, absoluteVersion); @@ -133,7 +102,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } @Override - protected void processDocs(RootDoc rootDoc, PrintStream ignore) { + protected void processDocs(RootDoc rootDoc) { // setup the global access to the root this.rootDoc = rootDoc; super.loadData(rootDoc, false); @@ -152,12 +121,12 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { // Specify how templates will see the data-model. This is an advanced topic... cfg.setObjectWrapper(new DefaultObjectWrapper()); - Set myWorkUnits = workUnits(); - for ( DocWorkUnit workUnit : myWorkUnits ) { + Set myWorkUnits = workUnits(); + for ( GATKDocWorkUnit workUnit : myWorkUnits ) { processDocWorkUnit(cfg, workUnit, myWorkUnits); } - processIndex(cfg, new ArrayList(myWorkUnits)); + processIndex(cfg, new ArrayList(myWorkUnits)); } catch ( FileNotFoundException e ) { throw new RuntimeException(e); } catch ( IOException e ) { @@ -214,7 +183,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return rootDoc.classNamed(clazz.getName()); } - private void processIndex(Configuration cfg, List indexData) throws IOException { + private void processIndex(Configuration cfg, List indexData) throws IOException { /* Get or create a template */ Template temp = cfg.getTemplate("generic.index.template.html"); @@ -228,7 +197,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { } } - private Map groupIndexData(List indexData) { + private Map groupIndexData(List indexData) { // // root -> data -> { summary -> y, filename -> z }, etc // -> groups -> group1, group2, etc. @@ -238,7 +207,7 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { Set docFeatures = new HashSet(); List> data = new ArrayList>(); - for ( DocWorkUnit workUnit : indexData ) { + for ( GATKDocWorkUnit workUnit : indexData ) { data.add(workUnit.toMap()); docFeatures.add(workUnit.annotation); } @@ -263,14 +232,14 @@ public class GATKDoclet extends ResourceBundleExtractorDoclet { return root; } - public final static DocWorkUnit findWorkUnitForClass(Class c, Set all) { - for ( final DocWorkUnit unit : all ) + public final static GATKDocWorkUnit findWorkUnitForClass(Class c, Set all) { + for ( final GATKDocWorkUnit unit : all ) if ( unit.clazz.equals(c) ) return unit; return null; } - private void processDocWorkUnit(Configuration cfg, DocWorkUnit unit, Set all) + private void processDocWorkUnit(Configuration cfg, GATKDocWorkUnit unit, Set all) throws IOException { System.out.printf("Processing documentation for class %s%n", unit.classDoc); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 5fd6f27d9..1aef110eb 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -32,7 +32,6 @@ import com.sun.javadoc.Tag; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.utils.Utils; -import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; @@ -43,9 +42,9 @@ import java.util.*; * */ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { - GATKDoclet.DocWorkUnit toProcess; + GATKDocWorkUnit toProcess; ClassDoc classdoc; - Set all; + Set all; RootDoc rootDoc; @Override @@ -66,7 +65,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } @Override - public void processOne(RootDoc rootDoc, GATKDoclet.DocWorkUnit toProcessArg, Set allArg) { + public void processOne(RootDoc rootDoc, GATKDocWorkUnit toProcessArg, Set allArg) { this.rootDoc = rootDoc; this.toProcess = toProcessArg; this.all = allArg; @@ -132,7 +131,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { // add in all of the explicitly related items for ( final Class extraDocClass : toProcess.annotation.extraDocs() ) { - final GATKDoclet.DocWorkUnit otherUnit = GATKDoclet.findWorkUnitForClass(extraDocClass, all); + final GATKDocWorkUnit otherUnit = GATKDoclet.findWorkUnitForClass(extraDocClass, all); if ( otherUnit == null ) throw new ReviewedStingException("Requested extraDocs for class without any documentation: " + extraDocClass); extraDocsData.add( @@ -143,7 +142,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } List> hierarchyDocs = new ArrayList>(); - for (final GATKDoclet.DocWorkUnit other : all ) { + for (final GATKDocWorkUnit other : all ) { final String relation = classRelationship(toProcess.clazz, other.clazz); if ( relation != null ) hierarchyDocs.add( From 1c1f1da349247654ea2c118720020a3e6ca0c9a9 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 20:01:59 -0400 Subject: [PATCH 37/41] Fixing compilation --- .../src/org/broadinstitute/sting/utils/help/GATKDoclet.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index ff20afbfa..2c8746f61 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -101,11 +101,9 @@ public class GATKDoclet { return m; } - @Override protected void processDocs(RootDoc rootDoc) { // setup the global access to the root this.rootDoc = rootDoc; - super.loadData(rootDoc, false); try { // basic setup From c43b5981f2507f2cdd8f1c59155a22e20f0d85d8 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 20:52:44 -0400 Subject: [PATCH 38/41] Hidden variables are hidden by default. Settable by command line option DiffObjectsWalker test arguments removed. Minor refactoring of GATKDoclet --- .../walkers/diffengine/DiffObjectsWalker.java | 15 --------------- .../help/DocumentedGATKFeatureHandler.java | 4 ++++ .../sting/utils/help/GATKDoclet.java | 9 ++++++++- .../help/GenericDocumentationHandler.java | 18 +++++++++++------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index c660bda36..fb7e6b579 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -182,21 +182,6 @@ public class DiffObjectsWalker extends RodWalker { @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) boolean showItemizedDifferences = false; - @Hidden - @Argument(fullName="testEnum", doc="X", required=false) - TestEnum testEnum = TestEnum.ONE; - - @Deprecated - @Argument(fullName="testDepreciates", doc="Y", required=false) - int dontUseMe = 1; - - public enum TestEnum { - /** Docs for ONE */ - ONE, - /** Docs for TWO */ - TWO - }; - final DiffEngine diffEngine = new DiffEngine(); @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java index 2227f9aba..366df0c3a 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/DocumentedGATKFeatureHandler.java @@ -44,6 +44,10 @@ public abstract class DocumentedGATKFeatureHandler { this.doclet = doclet; } + public GATKDoclet getDoclet() { + return doclet; + } + public boolean shouldBeProcessed(ClassDoc doc) { return true; } public String getDestinationFilename(ClassDoc doc) { diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index 2c8746f61..b4ab3b98e 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -45,6 +45,7 @@ public class GATKDoclet { final protected static File DESTINATION_DIR = new File("gatkdocs"); final protected static Logger logger = Logger.getLogger(GATKDoclet.class); protected static String buildTimestamp = null, absoluteVersion = null; + protected static boolean showHiddenFeatures = false; RootDoc rootDoc; @@ -61,6 +62,8 @@ public class GATKDoclet { buildTimestamp = options[1]; if (options[0].equals("-absolute-version")) absoluteVersion = options[1]; + if (options[0].equals("-include-hidden")) + showHiddenFeatures = true; } GATKDoclet doclet = new GATKDoclet(); @@ -74,12 +77,16 @@ public class GATKDoclet { * @return Number of potential parameters; 0 if not supported. */ public static int optionLength(String option) { - if(option.equals("-build-timestamp") || option.equals("-absolute-version") ) { + if(option.equals("-build-timestamp") || option.equals("-absolute-version") || option.equals("-include-hidden")) { return 2; } return 0; } + public boolean showHiddenFeatures() { + return showHiddenFeatures; + } + public Set workUnits() { TreeSet m = new TreeSet(); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 1aef110eb..8773fc4f8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -113,13 +113,17 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { ArgumentDefinition argDef = argumentSource.createArgumentDefinitions().get(0); FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); Map argBindings = docForArgument(fieldDoc, argumentSource, argDef); // todo -- why can you have multiple ones? - String kind = "optional"; - if ( argumentSource.isRequired() ) kind = "required"; - else if ( argumentSource.isHidden() ) kind = "hidden"; - else if ( argumentSource.isDeprecated() ) kind = "depreciated"; - args.get(kind).add(argBindings); - args.get("all").add(argBindings); - System.out.printf("Processing %s%n", argumentSource); + if ( ! argumentSource.isHidden() || getDoclet().showHiddenFeatures() ) { + System.out.printf("Processing %s%n", argumentSource); + String kind = "optional"; + if ( argumentSource.isRequired() ) kind = "required"; + else if ( argumentSource.isHidden() ) kind = "hidden"; + else if ( argumentSource.isDeprecated() ) kind = "depreciated"; + args.get(kind).add(argBindings); + args.get("all").add(argBindings); + } else { + System.out.printf("Skipping hidden feature %s%n", argumentSource); + } } } catch ( ClassNotFoundException e ) { throw new RuntimeException(e); From 2039ce6102538009aa526dc4469e9bc9b6884035 Mon Sep 17 00:00:00 2001 From: Mark DePristo Date: Sun, 24 Jul 2011 22:56:55 -0400 Subject: [PATCH 39/41] Default values now displayed in arguments DiffEngine fixed so that newInstance() would work. Pretty quickly encountered a situation where newInstance() failed. Debug output now written when this occurs in the log. Logger now used instead of standard out, with INFO the default level. --- .../walkers/diffengine/DiffObjectsWalker.java | 5 +- .../sting/utils/help/GATKDoclet.java | 7 +- .../help/GenericDocumentationHandler.java | 106 +++++++++++++++++- settings/helpTemplates/generic.template.html | 6 +- 4 files changed, 114 insertions(+), 10 deletions(-) diff --git a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java index fb7e6b579..b679f967a 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java +++ b/public/java/src/org/broadinstitute/sting/gatk/walkers/diffengine/DiffObjectsWalker.java @@ -35,6 +35,7 @@ import org.broadinstitute.sting.gatk.walkers.RodWalker; import java.io.File; import java.io.PrintStream; +import java.util.Arrays; import java.util.List; /** @@ -182,11 +183,11 @@ public class DiffObjectsWalker extends RodWalker { @Argument(fullName="showItemizedDifferences", shortName="SID", doc="Should we enumerate all differences between the files?", required=false) boolean showItemizedDifferences = false; - final DiffEngine diffEngine = new DiffEngine(); + DiffEngine diffEngine; @Override public void initialize() { - + this.diffEngine = new DiffEngine(); } @Override diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java index b4ab3b98e..0b4c69e3c 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GATKDoclet.java @@ -31,6 +31,7 @@ import freemarker.template.DefaultObjectWrapper; import freemarker.template.Template; import freemarker.template.TemplateException; import org.apache.commons.io.FileUtils; +import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; @@ -56,6 +57,7 @@ public class GATKDoclet { * @throws java.io.IOException if output can't be written. */ public static boolean start(RootDoc rootDoc) throws IOException { + logger.setLevel(Level.INFO); // load arguments for(String[] options: rootDoc.options()) { if(options[0].equals("-build-timestamp")) @@ -91,11 +93,12 @@ public class GATKDoclet { TreeSet m = new TreeSet(); for ( ClassDoc doc : rootDoc.classes() ) { - System.out.printf("Considering %s%n", doc); + logger.debug("Considering " + doc); Class clazz = getClassForClassDoc(doc); DocumentedGATKFeature feature = getFeatureForClassDoc(doc); DocumentedGATKFeatureHandler handler = createHandler(doc, feature); if ( handler != null && handler.shouldBeProcessed(doc) ) { + logger.info("Going to generate documentation for class " + doc); String filename = handler.getDestinationFilename(doc); GATKDocWorkUnit unit = new GATKDocWorkUnit(doc.name(), filename, feature.groupName(), @@ -246,7 +249,7 @@ public class GATKDoclet { private void processDocWorkUnit(Configuration cfg, GATKDocWorkUnit unit, Set all) throws IOException { - System.out.printf("Processing documentation for class %s%n", unit.classDoc); + //System.out.printf("Processing documentation for class %s%n", unit.classDoc); unit.handler.processOne(rootDoc, unit, all); diff --git a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java index 8773fc4f8..fd1048844 100644 --- a/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java +++ b/public/java/src/org/broadinstitute/sting/utils/help/GenericDocumentationHandler.java @@ -29,9 +29,11 @@ import com.sun.javadoc.ClassDoc; import com.sun.javadoc.FieldDoc; import com.sun.javadoc.RootDoc; import com.sun.javadoc.Tag; +import org.apache.log4j.Logger; import org.broadinstitute.sting.commandline.*; import org.broadinstitute.sting.gatk.CommandLineGATK; import org.broadinstitute.sting.utils.Utils; +import org.broadinstitute.sting.utils.classloader.JVMUtils; import org.broadinstitute.sting.utils.exceptions.ReviewedStingException; import java.io.*; @@ -42,6 +44,7 @@ import java.util.*; * */ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { + private static Logger logger = Logger.getLogger(GenericDocumentationHandler.class); GATKDocWorkUnit toProcess; ClassDoc classdoc; Set all; @@ -71,7 +74,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { this.all = allArg; this.classdoc = toProcess.classDoc; - System.out.printf("%s class %s%n", toProcess.group, toProcess.classDoc); + //System.out.printf("%s class %s%n", toProcess.group, toProcess.classDoc); Map root = new HashMap(); addHighLevelBindings(root); @@ -87,7 +90,7 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { // Extract overrides from the doc tags. StringBuilder summaryBuilder = new StringBuilder(); for(Tag tag: classdoc.firstSentenceTags()) - summaryBuilder.append(tag.text()); + summaryBuilder.append(tag.text()); root.put("summary", summaryBuilder.toString()); root.put("description", classdoc.commentText()); root.put("timestamp", toProcess.buildTimestamp); @@ -101,6 +104,9 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { protected void addArgumentBindings(Map root) { ParsingEngine parsingEngine = createStandardGATKParsingEngine(); + // attempt to instantiate the class + Object instance = makeInstanceIfPossible(toProcess.clazz); + Map> args = new HashMap>(); root.put("arguments", args); args.put("all", new ArrayList()); @@ -114,15 +120,23 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { FieldDoc fieldDoc = getFieldDoc(classdoc, argumentSource.field.getName()); Map argBindings = docForArgument(fieldDoc, argumentSource, argDef); // todo -- why can you have multiple ones? if ( ! argumentSource.isHidden() || getDoclet().showHiddenFeatures() ) { - System.out.printf("Processing %s%n", argumentSource); + logger.debug(String.format("Processing %s", argumentSource)); String kind = "optional"; if ( argumentSource.isRequired() ) kind = "required"; else if ( argumentSource.isHidden() ) kind = "hidden"; else if ( argumentSource.isDeprecated() ) kind = "depreciated"; + + // get the value of the field + if ( instance != null ) { + Object value = getFieldValue(toProcess.clazz, instance, fieldDoc.name()); + if ( value != null ) + argBindings.put("defaultValue", prettyPrintValueString(value)); + } + args.get(kind).add(argBindings); args.get("all").add(argBindings); } else { - System.out.printf("Skipping hidden feature %s%n", argumentSource); + logger.debug(String.format("Skipping hidden feature %s", argumentSource)); } } } catch ( ClassNotFoundException e ) { @@ -130,6 +144,90 @@ public class GenericDocumentationHandler extends DocumentedGATKFeatureHandler { } } + private Object getFieldValue(Class c, Object instance, String fieldName) { + Field field = JVMUtils.findField(c, fieldName); + if ( field != null ) { + Object value = JVMUtils.getFieldValue(field, instance); + //System.out.printf("Fetched value of field %s in class %s: %s%n", fieldName, c, value); + return value; + } else { + return findFieldValueInArgumentCollections(c, instance, fieldName); + } + } + + private Object findFieldValueInArgumentCollections(Class c, Object instance, String fieldName) { + for ( Field field : JVMUtils.getAllFields(c) ) { + if ( field.isAnnotationPresent(ArgumentCollection.class) ) { + //System.out.printf("Searching for %s in argument collection field %s%n", fieldName, field); + Object fieldValue = JVMUtils.getFieldValue(field, instance); + Object value = getFieldValue(fieldValue.getClass(), fieldValue, fieldName); + if ( value != null ) + return value; + } + } + + return null; + } + + /** + * Assumes value != null + * @param value + * @return + */ + private Object prettyPrintValueString(Object value) { + if ( value.getClass().isArray() ) { + Class type = value.getClass().getComponentType(); + if ( boolean.class.isAssignableFrom(type) ) + return Arrays.toString((boolean[])value); + if ( byte.class.isAssignableFrom(type) ) + return Arrays.toString((byte[])value); + if ( char.class.isAssignableFrom(type) ) + return Arrays.toString((char[])value); + if ( double.class.isAssignableFrom(type) ) + return Arrays.toString((double[])value); + if ( float.class.isAssignableFrom(type) ) + return Arrays.toString((float[])value); + if ( int.class.isAssignableFrom(type) ) + return Arrays.toString((int[])value); + if ( long.class.isAssignableFrom(type) ) + return Arrays.toString((long[])value); + if ( short.class.isAssignableFrom(type) ) + return Arrays.toString((short[])value); + if ( Object.class.isAssignableFrom(type) ) + return Arrays.toString((Object[])value); + else + throw new RuntimeException("Unexpected array type in prettyPrintValue. Value was " + value + " type is " + type); + } else + return value.toString(); + } + + private Object makeInstanceIfPossible(Class c) { + Object instance = null; + try { + // don't try to make something where we will obviously fail + if (! c.isEnum() && ! c.isAnnotation() && ! c.isAnonymousClass() && + ! c.isArray() && ! c.isPrimitive() & JVMUtils.isConcrete(c) ) { + instance = c.newInstance(); + //System.out.printf("Created object of class %s => %s%n", c, instance); + return instance; + } else + return null; + } + catch (IllegalAccessException e ) { } + catch (InstantiationException e ) { } + catch (ExceptionInInitializerError e ) { } + catch (SecurityException e ) { } + // this last one is super dangerous, but some of these methods catch ClassNotFoundExceptions + // and rethrow then as RuntimeExceptions + catch (RuntimeException e) {} + finally { + if ( instance == null ) + logger.warn(String.format("Unable to create instance of class %s => %s", c, instance)); + } + + return instance; + } + protected void addRelatedBindings(Map root) { List> extraDocsData = new ArrayList>(); diff --git a/settings/helpTemplates/generic.template.html b/settings/helpTemplates/generic.template.html index d2468214c..ca0d1e76f 100644 --- a/settings/helpTemplates/generic.template.html +++ b/settings/helpTemplates/generic.template.html @@ -2,11 +2,12 @@ <#macro argumentlist name myargs> <#if myargs?size != 0> - ${name} + ${name} <#list myargs as arg> ${arg.name} ${arg.type} + ${arg.defaultValue!"No default"} ${arg.summary} <#-- @@ -18,7 +19,7 @@ <#macro argumentDetails arg>

${arg.name}<#if arg.synonyms??> / ${arg.synonyms} - (<#if arg.attributes??>${arg.attributes} ${arg.type})

+ (<#if arg.attributes??>${arg.attributes} ${arg.type}<#if arg.defaultValue??> with default value ${arg.defaultValue}) ${arg.summary}. ${arg.fulltext}
<#if arg.options??>

The ${arg.name} argument is an enumerated type (${arg.type}), which can have one of the following values:

@@ -70,6 +71,7 @@ Name Type + Default value Summary From 4c6c16f89543d02cf171c18a4a17a1711f2de74c Mon Sep 17 00:00:00 2001 From: Mauricio Carneiro Date: Mon, 25 Jul 2011 00:25:08 -0400 Subject: [PATCH 40/41] Documented following the new gatkdoc framework --- .../sting/utils/ContigComparator.java | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java b/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java index 5e573418d..619beddb8 100644 --- a/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java +++ b/public/java/src/org/broadinstitute/sting/utils/ContigComparator.java @@ -9,7 +9,21 @@ import java.util.TreeSet; * User: carneiro * Date: 7/23/11 * Time: 6:07 PM - * To change this template use File | Settings | File Templates. + * + * Contig comparator -- sorting contigs like Picard + * + * This is very useful if you want to output your text files or manipulate data in the usual chromosome ordering : + * 1 + * 2 + * 3 + * ... + * 21 + * 22 + * X + * Y + * GL*** + * ... + * Just use this comparator in any SortedSet class constructor and your data will be sorted like in the BAM file. */ public class ContigComparator implements Comparator { private Set specialChrs;