A walker to analyze the memory consumption of reference, reads, and RODs at

each base both in bytes and as a percentage of the used heap size.  

May be a bit buggy at this point; there are a lot of metrics around the Java
heap and I'm not completely sure that the metrics I'm outputting are exactly
the ones that I'm looking for. 

Also fixed a documentation bug in my Sizeof class.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@5730 348d0f76-0448-11de-a6fe-93d51630548a

java/src/org/broadinstitute/sting/oneoffprojects/walkers/qc/AnalyzeMemoryConsumption.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.oneoffprojects.walkers.qc;
+
+import org.broadinstitute.sting.commandline.Argument;
+import org.broadinstitute.sting.commandline.Output;
+import org.broadinstitute.sting.gatk.contexts.AlignmentContext;
+import org.broadinstitute.sting.gatk.contexts.ReferenceContext;
+import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker;
+import org.broadinstitute.sting.gatk.walkers.LocusWalker;
+import org.broadinstitute.sting.utils.instrumentation.Sizeof;
+
+import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+
+/**
+ * Analyzes the memory consumption required by the input data set as a percentage of the total heap consumed.
+ * Uses the Sizeof operator, which means that some supplemental data must be added to the command line.
+ *
+ * add -javaagent:$STING_HOME/dist/StingUtils.jar as a command-line
+ * JVM argument.
+ *
+ * For up-to-the-minute documentation, see the org.broadinstitute.sting.utils.instrumentation.Sizeof class.
+ */
+public class AnalyzeMemoryConsumption extends LocusWalker<LocusContext,Long> {
+    @Output(doc="Write output to this file, or /dev/stdout if unspecified.")
+    private PrintStream out;
+
+    @Argument(doc="How frequently should we emit heap usage data",required=false)
+    private int frequency = 1000;
+
+    private MemoryMXBean monitor;
+
+    public void initialize() {
+        monitor = ManagementFactory.getMemoryMXBean();
+    }
+
+    public LocusContext map(RefMetaDataTracker tracker, ReferenceContext ref, AlignmentContext reads) {
+        return new LocusContext(ref,reads,tracker);
+    }
+
+    /**
+     *
+     */
+    public Long reduceInit() {
+        return 0L;
+    }
+
+    /**
+     *
+     */
+    public Long reduce(LocusContext locusContext, Long sum) {
+        sum++;
+
+        if(sum % frequency == 0) {
+            long refSize = Sizeof.getObjectGraphSize(locusContext.reference);
+            long readsSize = Sizeof.getObjectGraphSize(locusContext.alignedReads);
+            long trackerSize = Sizeof.getObjectGraphSize(locusContext.referenceOrderedData);
+
+            MemoryUsage memoryUsage = monitor.getHeapMemoryUsage();
+            long memoryUsed = memoryUsage.getUsed();
+            long maxMemory = memoryUsage.getMax();
+
+            out.printf("site: %s: reference: %d bytes (%f%%), reads: %d bytes (%f%%), reference-ordered data: %d bytes (%f%%); heap used = %d bytes (%f%%); max heap = %d bytes.%n",
+                    locusContext.reference.getLocus(),
+                    refSize,((float)refSize)/memoryUsed,
+                    readsSize,((float)readsSize)/memoryUsed,
+                    trackerSize,((float)trackerSize)/memoryUsed,
+                    memoryUsed,((float)memoryUsed)/maxMemory,
+                    maxMemory);
+        }
+
+        return sum;
+    }
+
+}
+
+/**
+ * Allows the user to easily pass data specific to a locus from map to reduce.
+ */
+class LocusContext {
+    public final ReferenceContext reference;
+    public final AlignmentContext alignedReads;
+    public final RefMetaDataTracker referenceOrderedData;
+
+    public LocusContext(final ReferenceContext reference, final AlignmentContext alignedReads, final RefMetaDataTracker referenceOrderedData) {
+        this.reference = reference;
+        this.alignedReads = alignedReads;
+        this.referenceOrderedData = referenceOrderedData;
+    }
+}

java/src/org/broadinstitute/sting/utils/instrumentation/Sizeof.java

@@ -36,7 +36,7 @@ import java.util.IdentityHashMap;
  * A sizeof implementation for Java.  Relies on the Java instrumentation API, so
  * it must be added as an agent to function properly.
  *
- * To run, add -javaagent:/Users/mhanna/src/Sting/dist/StingUtils.jar as a command-line
+ * To run, add -javaagent:$STING_HOME/dist/StingUtils.jar as a command-line
  * JVM argument.
  *
  * @author mhanna