From 0092a0b9eb81607a90e8f1fd4f3a329d46438bc4 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Sun, 2 Nov 2014 16:18:23 +0900 Subject: [PATCH] Faster builds, with updates to documentation generation. Reading the multiple GATKText files as a single stream, especially with new top level target executable jar files pointing to a lib folder. Don't dirty the build with a new GATKText.properties if input files are unmodified. Stop warning on undocumented abstract classes. Fixed ClassNotFoundException/NoClassDefFoundError by fixing ResourceBundleExtractorDoclet artifact. Excluding Exceptions from documentation. Removed custom log4j dependency from ResourceBundleExtractorDoclet. Stop generating the dependency reduced pom during shade. Stop regenerating gsalib when the files are already up to date. Disabled mvn site generation from external-example. --- pom.xml | 23 +- protected/gatk-tools-protected/pom.xml | 10 - .../tools/walkers/indels/IndelRealigner.java | 5 +- .../SimulateReadsForVariants.java | 4 +- public/external-example/pom.xml | 11 +- public/gatk-engine/pom.xml | 10 - .../gatk/engine/WalkerManager.java | 2 +- .../gatk/engine/io/NWaySAMFileWriter.java | 6 +- .../gatk/queue/QCommandLine.scala | 27 +-- public/gatk-tools-public/pom.xml | 10 - public/gatk-utils/pom.xml | 10 - .../gatk/utils/help/log4j.properties | 7 - .../org/broadinstitute/gatk/utils/Utils.java | 74 +++++-- .../utils/commandline/CommandLineProgram.java | 5 +- .../gatk/utils/help/DocletUtils.java | 12 +- .../help/ResourceBundleExtractorDoclet.java | 203 +++++++++++------- .../gatk/utils/io/Resource.java | 51 +++++ .../gatk/utils/text/TextFormattingUtils.java | 18 +- public/gsalib/pom.xml | 8 + 19 files changed, 282 insertions(+), 214 deletions(-) delete mode 100644 public/gatk-utils/src/main/config/org/broadinstitute/gatk/utils/help/log4j.properties diff --git a/pom.xml b/pom.xml index 5272f6c55..95440ec8a 100644 --- a/pom.xml +++ b/pom.xml @@ -165,7 +165,6 @@ - org.apache.maven.plugins maven-resources-plugin @@ -184,25 +183,6 @@ ${gatk.process-test-resources.phase} - - copy-resource-bundle-log4j - - copy-resources - - none - - - ${project.reporting.outputDirectory}/apidocs - - - ${gatk.basedir}/public/gatk-utils/src/main/config/org/broadinstitute/gatk/utils/help - - - - @@ -223,7 +203,7 @@ ${project.build.outputDirectory} ${project.groupId} - gatk-utils + ${project.artifactId} ${project.version} 2g @@ -402,6 +382,7 @@ none true + false org.broadinstitute.gatk:gsalib:tar.gz:* diff --git a/protected/gatk-tools-protected/pom.xml b/protected/gatk-tools-protected/pom.xml index 24ceffe87..3df22c1a1 100644 --- a/protected/gatk-tools-protected/pom.xml +++ b/protected/gatk-tools-protected/pom.xml @@ -71,16 +71,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resource-bundle-log4j - prepare-package - - - org.apache.maven.plugins maven-javadoc-plugin diff --git a/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/indels/IndelRealigner.java b/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/indels/IndelRealigner.java index 9806a92e9..6cd78be45 100644 --- a/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/indels/IndelRealigner.java +++ b/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/indels/IndelRealigner.java @@ -81,7 +81,6 @@ import org.broadinstitute.gatk.utils.sam.AlignmentUtils; import org.broadinstitute.gatk.utils.sam.GATKSAMRecord; import org.broadinstitute.gatk.engine.io.NWaySAMFileWriter; import org.broadinstitute.gatk.utils.sam.ReadUtils; -import org.broadinstitute.gatk.utils.text.TextFormattingUtils; import org.broadinstitute.gatk.utils.text.XReadLines; import htsjdk.variant.variantcontext.VariantContext; @@ -476,10 +475,8 @@ public class IndelRealigner extends ReadWalker { if ( NO_PG_TAG ) return null; final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); - final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("StingText"); try { - final String version = headerInfo.getString("org.broadinstitute.sting.gatk.version"); - programRecord.setProgramVersion(version); + programRecord.setProgramVersion(CommandLineProgram.getVersionNumber()); } catch (MissingResourceException e) { // this is left empty on purpose (perhaps Andrey knows why?) } diff --git a/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/simulatereads/SimulateReadsForVariants.java b/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/simulatereads/SimulateReadsForVariants.java index 0bc14bc49..d8350d9e4 100644 --- a/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/simulatereads/SimulateReadsForVariants.java +++ b/protected/gatk-tools-protected/src/main/java/org/broadinstitute/gatk/tools/walkers/simulatereads/SimulateReadsForVariants.java @@ -72,7 +72,6 @@ import org.broadinstitute.gatk.utils.*; import org.broadinstitute.gatk.utils.exceptions.UserException; import htsjdk.variant.variantcontext.*; import org.broadinstitute.gatk.utils.sam.GATKSAMRecord; -import org.broadinstitute.gatk.utils.text.TextFormattingUtils; import org.broadinstitute.gatk.utils.help.DocumentedGATKFeature; import org.broadinstitute.gatk.utils.help.HelpConstants; import htsjdk.variant.vcf.VCFConstants; @@ -222,8 +221,7 @@ public class SimulateReadsForVariants extends RodWalker { final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); if ( !NO_PG_TAG ) { - final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("GATKText"); - programRecord.setProgramVersion(headerInfo.getString("org.broadinstitute.gatk.tools.version")); + programRecord.setProgramVersion(CommandLineProgram.getVersionNumber()); programRecord.setCommandLine(getToolkit().createApproximateCommandLineArgumentString(getToolkit(), this)); } header.setProgramRecords(Arrays.asList(programRecord)); diff --git a/public/external-example/pom.xml b/public/external-example/pom.xml index aa4e57cc4..67535a788 100644 --- a/public/external-example/pom.xml +++ b/public/external-example/pom.xml @@ -119,9 +119,9 @@ ${project.build.outputDirectory} - org.broadinstitute.gatk - gatk-utils - ${gatk.version} + ${project.groupId} + ${project.artifactId} + ${project.version} 2g false @@ -145,6 +145,7 @@ true + false @@ -259,6 +260,10 @@ + + true + + diff --git a/public/gatk-engine/pom.xml b/public/gatk-engine/pom.xml index 6d2696c7a..1f59cd1bc 100644 --- a/public/gatk-engine/pom.xml +++ b/public/gatk-engine/pom.xml @@ -50,16 +50,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resource-bundle-log4j - prepare-package - - - org.apache.maven.plugins maven-javadoc-plugin diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/WalkerManager.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/WalkerManager.java index 9ea5a3c31..81e22c247 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/WalkerManager.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/WalkerManager.java @@ -54,7 +54,7 @@ public class WalkerManager extends PluginManager { public WalkerManager() { super(Walker.class,"walker",""); - helpText = TextFormattingUtils.loadResourceBundle("GATKText"); + helpText = TextFormattingUtils.GATK_RESOURCE_BUNDLE; } /** diff --git a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/io/NWaySAMFileWriter.java b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/io/NWaySAMFileWriter.java index 011963ecc..74ed19d3e 100644 --- a/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/io/NWaySAMFileWriter.java +++ b/public/gatk-engine/src/main/java/org/broadinstitute/gatk/engine/io/NWaySAMFileWriter.java @@ -28,11 +28,11 @@ package org.broadinstitute.gatk.engine.io; import htsjdk.samtools.*; import htsjdk.samtools.util.ProgressLoggerInterface; import org.broadinstitute.gatk.engine.GenomeAnalysisEngine; +import org.broadinstitute.gatk.utils.commandline.CommandLineProgram; import org.broadinstitute.gatk.utils.sam.SAMReaderID; import org.broadinstitute.gatk.utils.exceptions.GATKException; import org.broadinstitute.gatk.utils.exceptions.UserException; import org.broadinstitute.gatk.utils.sam.GATKSAMFileWriter; -import org.broadinstitute.gatk.utils.text.TextFormattingUtils; import java.io.File; import java.util.*; @@ -141,10 +141,8 @@ public class NWaySAMFileWriter implements SAMFileWriter { */ public static SAMProgramRecord createProgramRecord(GenomeAnalysisEngine toolkit, Object walker, String PROGRAM_RECORD_NAME) { final SAMProgramRecord programRecord = new SAMProgramRecord(PROGRAM_RECORD_NAME); - final ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("GATKText"); try { - final String version = headerInfo.getString("org.broadinstitute.gatk.engine.version"); - programRecord.setProgramVersion(version); + programRecord.setProgramVersion(CommandLineProgram.getVersionNumber()); } catch (MissingResourceException e) { // couldn't care less if the resource is missing... } diff --git a/public/gatk-queue/src/main/scala/org/broadinstitute/gatk/queue/QCommandLine.scala b/public/gatk-queue/src/main/scala/org/broadinstitute/gatk/queue/QCommandLine.scala index 297e10bb3..843743e48 100644 --- a/public/gatk-queue/src/main/scala/org/broadinstitute/gatk/queue/QCommandLine.scala +++ b/public/gatk-queue/src/main/scala/org/broadinstitute/gatk/queue/QCommandLine.scala @@ -34,8 +34,7 @@ import org.broadinstitute.gatk.utils.classloader.PluginManager import org.broadinstitute.gatk.utils.exceptions.UserException import org.broadinstitute.gatk.utils.io.IOUtils import org.broadinstitute.gatk.utils.help.ApplicationDetails -import java.util.{ResourceBundle, Arrays} -import org.broadinstitute.gatk.utils.text.TextFormattingUtils +import java.util.Arrays import org.apache.commons.io.FilenameUtils /** @@ -260,33 +259,11 @@ class QCommandLine extends CommandLineProgram with Logging { } private def createQueueHeader() : Seq[String] = { - Seq(String.format("Queue v%s, Compiled %s", getQueueVersion, getBuildTimestamp), + Seq(String.format("Queue v%s, Compiled %s", CommandLineProgram.getVersionNumber, CommandLineProgram.getBuildTime), "Copyright (c) 2012 The Broad Institute", "For support and documentation go to http://www.broadinstitute.org/gatk") } - private def getQueueVersion : String = { - val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("GATKText") - - if ( stingResources.containsKey("org.broadinstitute.gatk.queue.QueueVersion.version") ) { - stingResources.getString("org.broadinstitute.gatk.queue.QueueVersion.version") - } - else { - "" - } - } - - private def getBuildTimestamp : String = { - val stingResources : ResourceBundle = TextFormattingUtils.loadResourceBundle("GATKText") - - if ( stingResources.containsKey("build.timestamp") ) { - stingResources.getString("build.timestamp") - } - else { - "" - } - } - def shutdown() = { shuttingDown = true qGraph.shutdown() diff --git a/public/gatk-tools-public/pom.xml b/public/gatk-tools-public/pom.xml index 0a5755026..cbf26bb41 100644 --- a/public/gatk-tools-public/pom.xml +++ b/public/gatk-tools-public/pom.xml @@ -54,16 +54,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resource-bundle-log4j - prepare-package - - - org.apache.maven.plugins maven-javadoc-plugin diff --git a/public/gatk-utils/pom.xml b/public/gatk-utils/pom.xml index e1aae406a..e0e2fad72 100644 --- a/public/gatk-utils/pom.xml +++ b/public/gatk-utils/pom.xml @@ -137,16 +137,6 @@ - - org.apache.maven.plugins - maven-resources-plugin - - - copy-resource-bundle-log4j - prepare-package - - - org.apache.maven.plugins maven-javadoc-plugin diff --git a/public/gatk-utils/src/main/config/org/broadinstitute/gatk/utils/help/log4j.properties b/public/gatk-utils/src/main/config/org/broadinstitute/gatk/utils/help/log4j.properties deleted file mode 100644 index 38c8335c9..000000000 --- a/public/gatk-utils/src/main/config/org/broadinstitute/gatk/utils/help/log4j.properties +++ /dev/null @@ -1,7 +0,0 @@ -# Root logger option -log4j.rootLogger=INFO, stdout - -# Direct log messages to stdout -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.Target=System.out -log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/Utils.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/Utils.java index 408fd9e9f..4641e0542 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/Utils.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/Utils.java @@ -53,6 +53,14 @@ public class Utils { public static void resetRandomGenerator() { randomGenerator.setSeed(GATK_RANDOM_SEED); } public static void resetRandomGenerator(long seed) { randomGenerator.setSeed(seed); } + private static final int TEXT_WARNING_WIDTH = 68; + private static final String TEXT_WARNING_PREFIX = "* "; + private static final String TEXT_WARNING_BORDER = dupString('*', TEXT_WARNING_PREFIX.length() + TEXT_WARNING_WIDTH); + private static final char ESCAPE_CHAR = '\u001B'; + // ASCII codes for making text blink + public static final String TEXT_BLINK = ESCAPE_CHAR + "[5m"; + public static final String TEXT_RESET = ESCAPE_CHAR + "[m"; + /** our log, which we want to capture anything from this class */ private static Logger logger = Logger.getLogger(Utils.class); @@ -106,28 +114,66 @@ public class Utils { } public static void warnUser(final Logger logger, final String msg) { - logger.warn(String.format("********************************************************************************")); - logger.warn(String.format("* WARNING:")); - logger.warn(String.format("*")); - prettyPrintWarningMessage(logger, msg); - logger.warn(String.format("********************************************************************************")); + for (final String line: warnUserLines(msg)) + logger.warn(line); + } + + public static List warnUserLines(final String msg) { + List results = new ArrayList<>(); + results.add(String.format(TEXT_WARNING_BORDER)); + results.add(String.format(TEXT_WARNING_PREFIX + "WARNING:")); + results.add(String.format(TEXT_WARNING_PREFIX)); + prettyPrintWarningMessage(results, msg); + results.add(String.format(TEXT_WARNING_BORDER)); + return results; } /** * pretty print the warning message supplied * - * @param logger logger for the message + * @param results the pretty printed message * @param message the message */ - private static void prettyPrintWarningMessage(Logger logger, String message) { - StringBuilder builder = new StringBuilder(message); - while (builder.length() > 70) { - int space = builder.lastIndexOf(" ", 70); - if (space <= 0) space = 70; - logger.warn(String.format("* %s", builder.substring(0, space))); - builder.delete(0, space + 1); + private static void prettyPrintWarningMessage(final List results, final String message) { + for (final String line: message.split("\\r?\\n")) { + final StringBuilder builder = new StringBuilder(line); + while (builder.length() > TEXT_WARNING_WIDTH) { + int space = getLastSpace(builder, TEXT_WARNING_WIDTH); + if (space <= 0) space = TEXT_WARNING_WIDTH; + results.add(String.format("%s%s", TEXT_WARNING_PREFIX, builder.substring(0, space))); + builder.delete(0, space + 1); + } + results.add(String.format("%s%s", TEXT_WARNING_PREFIX, builder)); } - logger.warn(String.format("* %s", builder)); + } + + /** + * Returns the last whitespace location in string, before width characters. + * @param message The message to break. + * @param width The width of the line. + * @return The last whitespace location. + */ + private static int getLastSpace(final CharSequence message, int width) { + final int length = message.length(); + int stopPos = width; + int currPos = 0; + int lastSpace = -1; + boolean inEscape = false; + while (currPos < stopPos && currPos < length) { + final char c = message.charAt(currPos); + if (c == ESCAPE_CHAR) { + stopPos++; + inEscape = true; + } else if (inEscape) { + stopPos++; + if (Character.isLetter(c)) + inEscape = false; + } else if (Character.isWhitespace(c)) { + lastSpace = currPos; + } + currPos++; + } + return lastSpace; } /** diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/commandline/CommandLineProgram.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/commandline/CommandLineProgram.java index 0c3cbecc7..70c06e24a 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/commandline/CommandLineProgram.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/commandline/CommandLineProgram.java @@ -296,13 +296,12 @@ public abstract class CommandLineProgram { } public static String getVersionNumber() { - // TODO: Confirm that version is available elsewhere not on tools. - ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("GATKText"); + ResourceBundle headerInfo = TextFormattingUtils.GATK_RESOURCE_BUNDLE; return headerInfo.containsKey("org.broadinstitute.gatk.utils.version") ? headerInfo.getString("org.broadinstitute.gatk.utils.version") : ""; } public static String getBuildTime() { - ResourceBundle headerInfo = TextFormattingUtils.loadResourceBundle("GATKText"); + ResourceBundle headerInfo = TextFormattingUtils.GATK_RESOURCE_BUNDLE; return headerInfo.containsKey("build.timestamp") ? headerInfo.getString("build.timestamp") : ""; } diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/DocletUtils.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/DocletUtils.java index 324fcfc84..4b94e019d 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/DocletUtils.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/DocletUtils.java @@ -49,7 +49,7 @@ public class DocletUtils { } protected static Class getClassForDoc(ProgramElementDoc doc) throws ClassNotFoundException { - return Class.forName(getClassName(doc)); + return Class.forName(getClassName(doc, true)); } protected static Field getFieldForFieldDoc(FieldDoc fieldDoc) { @@ -67,10 +67,14 @@ public class DocletUtils { * @param doc the Javadoc model for the given class. * @return The (string) class name of the given class. */ - protected static String getClassName(ProgramElementDoc doc) { + protected static String getClassName(ProgramElementDoc doc, boolean binaryName) { PackageDoc containingPackage = doc.containingPackage(); + String className = doc.name(); + if (binaryName) { + className = className.replaceAll("\\.", "\\$"); + } return containingPackage.name().length() > 0 ? - String.format("%s.%s", containingPackage.name(), doc.name()) : - String.format("%s", doc.name()); + String.format("%s.%s", containingPackage.name(), className) : + String.format("%s", className); } } \ No newline at end of file diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/ResourceBundleExtractorDoclet.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/ResourceBundleExtractorDoclet.java index f225f4197..3de5484aa 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/ResourceBundleExtractorDoclet.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/help/ResourceBundleExtractorDoclet.java @@ -39,6 +39,11 @@ import java.util.*; * @version 0.1 */ public class ResourceBundleExtractorDoclet { + // NOTE: Using log4j during javadoc generation requires + // a proper Log4J initialization (see CommandLineProgram), + // or a log4.properties file. This doclet has neither. + //private static Logger logger = Logger.getLogger(ResourceBundleExtractorDoclet.class); + /** * Taglet for the particular version number. */ @@ -46,16 +51,12 @@ public class ResourceBundleExtractorDoclet { public static final String SUMMARY_TAGLET_NAME = "help.summary"; public static final String DESCRIPTION_TAGLET_NAME = "help.description"; - /** - * Maintains a collection of resources in memory as they're accumulated. - */ - protected final Properties resourceText = new Properties(); - - /** - * Maintains a collection of classes that should really be documented. - */ - protected final Set undocumentedClasses = new HashSet(); + private final RootDoc rootDoc; + private final Set classDocs; + private final Set packageDocs; + private final Set allDocs; + protected File outFile = null; protected String buildTimestamp = null, absoluteVersion = null; /** @@ -65,75 +66,22 @@ public class ResourceBundleExtractorDoclet { * @throws IOException if output can't be written. */ public static boolean start(RootDoc rootDoc) throws IOException { - ResourceBundleExtractorDoclet doclet = new ResourceBundleExtractorDoclet(); - PrintStream out = doclet.loadData(rootDoc, true); - doclet.processDocs(rootDoc, out); + ResourceBundleExtractorDoclet doclet = new ResourceBundleExtractorDoclet(rootDoc); + doclet.checkUndocumentedClasses(); + if (doclet.isUpToDate()) { + rootDoc.printNotice("Docs up to date. Not regenerating."); + return true; + } + doclet.processDocs(); return true; } - protected PrintStream loadData(RootDoc rootDoc, boolean overwriteResourcesFile) { - PrintStream out = System.out; - - for(String[] options: rootDoc.options()) { - if(options[0].equals("-out")) { - try { - loadExistingResourceFile(options[1], rootDoc); - if ( overwriteResourcesFile ) - out = new PrintStream(options[1]); - } catch ( FileNotFoundException e ) { - throw new RuntimeException(e); - } catch ( IOException e ) { - throw new RuntimeException(e); - } - } - if(options[0].equals("-build-timestamp")) - buildTimestamp = options[1]; - if (options[0].equals("-absolute-version")) - absoluteVersion = options[1]; - } - - resourceText.setProperty("build.timestamp",buildTimestamp); - return out; - } - - protected void processDocs(RootDoc rootDoc, PrintStream out) { - // Cache packages as we see them, since there's no direct way to iterate over packages. - Set packages = new HashSet(); - - for(ClassDoc currentClass: rootDoc.classes()) { - PackageDoc containingPackage = currentClass.containingPackage(); - packages.add(containingPackage); - - if(isRequiredJavadocMissing(currentClass) && shouldDocument(currentClass)) - undocumentedClasses.add(currentClass.name()); - - renderHelpText(DocletUtils.getClassName(currentClass),currentClass); - } - - for(PackageDoc currentPackage: packages) - renderHelpText(currentPackage.name(),currentPackage); - - try { - resourceText.store(out,"Strings displayed by the GATK help system"); - } catch ( FileNotFoundException e ) { - throw new RuntimeException(e); - } catch ( IOException e ) { - throw new RuntimeException(e); - } - - // ASCII codes for making text blink - final String blink = "\u001B\u005B\u0035\u006D"; - final String reset = "\u001B\u005B\u006D"; - - if(undocumentedClasses.size() > 0) - Utils.warnUser(String.format("The following are currently undocumented: %s%s%s", blink, Utils.join(" ", undocumentedClasses), reset)); - } - /** * Validate the given options against options supported by this doclet. * @param option Option to validate. * @return Number of potential parameters; 0 if not supported. */ + @SuppressWarnings("unused") // Used by javadoc system public static int optionLength(String option) { if(option.equals("-build-timestamp") || option.equals("-out") || option.equals("-absolute-version") ) { return 2; @@ -141,25 +89,111 @@ public class ResourceBundleExtractorDoclet { return 0; } + /** + * Creates a new resource extractor doclet. + * @param rootDoc the documentation root. + */ + private ResourceBundleExtractorDoclet(RootDoc rootDoc) { + this.rootDoc = rootDoc; + this.classDocs = new TreeSet<>(); + this.packageDocs = new TreeSet<>(); + this.allDocs = new TreeSet<>(); + for (final ClassDoc classDoc: rootDoc.classes()) { + this.classDocs.add(classDoc); + // Cache packages as we see them, since there's no direct way to iterate over packages. + this.packageDocs.add(classDoc.containingPackage()); + } + this.allDocs.addAll(classDocs); + this.allDocs.addAll(packageDocs); + for(final String[] options: rootDoc.options()) { + if(options[0].equals("-out")) + this.outFile = new File(options[1]); + if(options[0].equals("-build-timestamp")) + this.buildTimestamp = options[1]; + if (options[0].equals("-absolute-version")) + this.absoluteVersion = options[1]; + } + } + + private void checkUndocumentedClasses() { + final Set undocumentedClasses = new TreeSet<>(); + + for (final ClassDoc classDoc: classDocs) { + if(isRequiredJavadocMissing(classDoc) && shouldDocument(classDoc)) + undocumentedClasses.add(classDoc.name()); + } + + if(undocumentedClasses.size() > 0) { + final String message = String.format("The following are currently undocumented: %s%s%s", + Utils.TEXT_BLINK, Utils.join(" ", undocumentedClasses), Utils.TEXT_RESET); + for (final String line: Utils.warnUserLines(message)) { + rootDoc.printWarning(line); + } + } + } + + private boolean isUpToDate() { + if (outFile == null) + return false; + + final long outFileMillis = outFile.lastModified(); + + if (outFileMillis == 0L) { + return false; + } + + for (final Doc doc: allDocs) { + final File docFile = doc.position() == null ? null : doc.position().file(); + if (docFile != null && docFile.lastModified() > outFileMillis) { + rootDoc.printNotice("At least one item is out of date: " + docFile.getAbsolutePath()); + return false; + } + } + + return true; + } + + protected void processDocs() throws IOException { + final PrintStream out; + if (outFile != null) { + out = new PrintStream(outFile); + } else { + out = System.out; + } + try { + // Maintains a collection of resources in memory as they're accumulated. + final Properties resourceText = new Properties(); + + loadExistingResourceFile(resourceText); + + resourceText.setProperty("build.timestamp", buildTimestamp); + + for (final ClassDoc currentClass : classDocs) + renderHelpText(resourceText, DocletUtils.getClassName(currentClass, false), currentClass); + for (final PackageDoc currentPackage : packageDocs) + renderHelpText(resourceText, currentPackage.name(), currentPackage); + + resourceText.store(out, "Strings displayed by the GATK help system"); + } finally { + if (outFile != null) { + out.close(); + } + } + } + /** * Attempts to load the contents of the resource file named by resourceFileName into * our in-memory resource collection resourceText. If the resource file doesn't exist, * prints a notice to the user but does not throw an exception back to the calling method, * since we'll just create a new resource file from scratch in that case. - * @param resourceFileName name of the resource file to attempt to load. - * @param rootDoc the documentation root. * @throws IOException if there is an I/O-related error other than FileNotFoundException * while attempting to read the resource file. */ - private void loadExistingResourceFile( String resourceFileName, RootDoc rootDoc ) throws IOException { + private void loadExistingResourceFile(final Properties resourceText) throws IOException { try { - BufferedReader resourceFile = new BufferedReader(new FileReader(resourceFileName)); - try { + try (final BufferedReader resourceFile = new BufferedReader(new FileReader(outFile))) { resourceText.load(resourceFile); } - finally { - resourceFile.close(); - } } catch ( FileNotFoundException e ) { rootDoc.printNotice("Resource file not found -- generating a new one from scratch."); @@ -172,11 +206,14 @@ public class ResourceBundleExtractorDoclet { * @return True if the class should be documented. False otherwise. */ protected static boolean shouldDocument(ClassDoc classDoc) { + if (classDoc.isAbstract()) { + return false; + } // TODO: Code duplication with GATKDoclet, including DocletUtils.getClassForDoc(). // TODO: Refactor common methods into DocletUtils, and possibly just use DocumentGATKFeatureObjects. - final Class docClass; + final Class docClass; try { - docClass = (Class) DocletUtils.getClassForDoc(classDoc); + docClass = (Class) DocletUtils.getClassForDoc(classDoc); } catch (ClassNotFoundException e) { return false; } catch (NoClassDefFoundError e) { @@ -184,6 +221,9 @@ public class ResourceBundleExtractorDoclet { } catch (UnsatisfiedLinkError e) { return false; // naughty BWA bindings } + if (Throwable.class.isAssignableFrom(docClass)) { + return false; // UserExceptions + } final DocumentedGATKFeature f = docClass.getAnnotation(DocumentedGATKFeature.class); return f != null && f.enable(); } @@ -199,10 +239,11 @@ public class ResourceBundleExtractorDoclet { /** * Renders all the help text required for a given name. + * @param resourceText resource text properties * @param elementName element name to use as the key * @param element Doc element to process. */ - private void renderHelpText(String elementName, Doc element) { + private void renderHelpText(final Properties resourceText, final String elementName, final Doc element) { StringBuilder summaryBuilder = new StringBuilder(); for(Tag tag: element.firstSentenceTags()) summaryBuilder.append(tag.text()); diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/io/Resource.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/io/Resource.java index abebe5299..c5a1cc4d8 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/io/Resource.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/io/Resource.java @@ -26,7 +26,14 @@ package org.broadinstitute.gatk.utils.io; import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Enumeration; +import java.util.List; /** * Stores a resource by path and a relative class. @@ -34,6 +41,7 @@ import java.io.InputStream; public class Resource { private final String path; private final Class relativeClass; + private final ClassLoader relativeClassLoader; /** * Create a resource with a path and a relative class. @@ -45,12 +53,20 @@ public class Resource { public Resource(String path, Class relativeClass) { this.path = path; this.relativeClass = relativeClass; + ClassLoader classLoader = null; + if (relativeClass != null) + classLoader = relativeClass.getClassLoader(); + this.relativeClassLoader = classLoader != null ? classLoader : ClassLoader.getSystemClassLoader(); } public Class getRelativeClass() { return relativeClass; } + public ClassLoader getRelativeClassLoader() { + return relativeClassLoader; + } + public String getPath() { return path; } @@ -88,4 +104,39 @@ public class Resource { return inputStream; } + + /** + * Get the contents of this resource as an InputStream + * @throws IllegalArgumentException if resource cannot be read + * @return an input stream that will read the contents of these resources + */ + public List getAllResourcesContentsAsStreams() { + final List resourceStreams = new ArrayList(); + try { + final Enumeration resources = getRelativeClassLoader().getResources(path); + while (resources.hasMoreElements()) { + try { + resourceStreams.add(resources.nextElement().openStream()); + } catch (IOException ignored) { + /* skip exceptions, just like ClassLoader.getSystemResourceAsStream() */ + } + } + } catch (IOException ignoredAlso) { + /* skip exceptions, just like ClassLoader.getSystemResourceAsStream() */ + } + if (resourceStreams.isEmpty()) { + throw new IllegalArgumentException("Resource not found: " + path); + } + return resourceStreams; + } + + /** + * Get the contents of this resource as an InputStream + * @throws IllegalArgumentException if resource cannot be read + * @return an input stream that will read the contents of these resources + */ + public InputStream getAllResourcesContentsAsStream() { + final List resourceStreams = getAllResourcesContentsAsStreams(); + return new SequenceInputStream(Collections.enumeration(resourceStreams)); + } } diff --git a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/text/TextFormattingUtils.java b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/text/TextFormattingUtils.java index 65fb970e1..b4409a919 100644 --- a/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/text/TextFormattingUtils.java +++ b/public/gatk-utils/src/main/java/org/broadinstitute/gatk/utils/text/TextFormattingUtils.java @@ -27,6 +27,7 @@ package org.broadinstitute.gatk.utils.text; import org.apache.log4j.Logger; import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException; +import org.broadinstitute.gatk.utils.io.Resource; import java.io.IOException; import java.io.StringReader; @@ -44,7 +45,12 @@ public class TextFormattingUtils { /** * our log, which we want to capture anything from this class */ - private static Logger logger = Logger.getLogger(TextFormattingUtils.class); + private static Logger logger = Logger.getLogger(TextFormattingUtils.class); + + /** + * The contents of the GATK bundle. If no such resource exists, warn the user and create an empty bundle. + */ + public static final ResourceBundle GATK_RESOURCE_BUNDLE = loadResourceBundle("GATKText", null); /** * The default line width, for GATK output written to the screen. @@ -96,14 +102,18 @@ public class TextFormattingUtils { * Load the contents of a resource bundle with the given name. If no such resource exists, warn the user * and create an empty bundle. * @param bundleName The name of the bundle to load. + * @param relativeClass The relative class or null to load a bundle from the root. * @return The best resource bundle that can be found matching the given name. */ - public static ResourceBundle loadResourceBundle(String bundleName) { + public static ResourceBundle loadResourceBundle(String bundleName, Class relativeClass) { + final ResourceBundle.Control c = ResourceBundle.Control.getControl(ResourceBundle.Control.FORMAT_DEFAULT); + final String resourceName = c.toResourceName(c.toBundleName(bundleName, Locale.ROOT), "properties"); + final Resource resource = new Resource(resourceName, relativeClass); ResourceBundle bundle; try { - bundle = ResourceBundle.getBundle(bundleName); + bundle = new PropertyResourceBundle(resource.getAllResourcesContentsAsStream()); } - catch(MissingResourceException ex) { + catch(Exception ex) { //logger.warn("Unable to load help text. Help output will be sparse."); // Generate an empty resource bundle. try { diff --git a/public/gsalib/pom.xml b/public/gsalib/pom.xml index f3dcf4f24..2000c408b 100644 --- a/public/gsalib/pom.xml +++ b/public/gsalib/pom.xml @@ -32,6 +32,14 @@ ${gatk.generate-resources.phase} + + true false src/assembly/gsalib.xml