Merge branch 'master' of ssh://nickel.broadinstitute.org/humgen/gsa-scr1/gsa-engineering/git/unstable

public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java

@@ -46,7 +46,6 @@ import org.simpleframework.xml.stream.Format;
 import org.simpleframework.xml.stream.HyphenStyle;
 
 import java.io.*;
-import java.net.InetAddress;
 import java.security.NoSuchAlgorithmException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
@@ -230,22 +229,6 @@ public class GATKRunReport {
     }
 
 
-    /**
-     * Helper utility that calls into the InetAddress system to resolve the hostname.  If this fails,
-     * unresolvable gets returned instead.
-     *
-     * @return
-     */
-    private String resolveHostname() {
-        try {
-            return InetAddress.getLocalHost().getCanonicalHostName();
-        }
-        catch (java.net.UnknownHostException uhe) { // [beware typo in code sample -dmw]
-            return "unresolvable";
-            // handle exception
-        }
-    }
-
     public void postReport(PhoneHomeOption type) {
         logger.debug("Posting report of type " + type);
         switch (type) {
@@ -325,7 +308,7 @@ public class GATKRunReport {
 
     private void postReportToAWSS3() {
         // modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html
-        this.hostName = resolveHostname(); // we want to fill in the host name
+        this.hostName = Utils.resolveHostname(); // we want to fill in the host name
         File localFile = postReportToLocalDisk(new File("./"));
         logger.debug("Generating GATK report to AWS S3 based on local file " + localFile);
         if ( localFile != null ) { // we succeeded in creating the local file

public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java

@@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks;
 
 import net.sf.samtools.SAMSequenceDictionary;
 import net.sf.samtools.util.CloseableIterator;
+import org.apache.log4j.Logger;
 import org.broad.tribble.FeatureCodec;
 import org.broad.tribble.FeatureSource;
 import org.broadinstitute.sting.gatk.refdata.utils.FeatureToGATKFeatureIterator;
@@ -45,10 +46,11 @@ import java.io.IOException;
  *         the basics of what a reference metadata track must contain.
  */
 public class RMDTrack {
+    private final static Logger logger = Logger.getLogger(RMDTrackBuilder.class);
+    private final static boolean DEBUG = false;
 
     // the basics of a track:
     private final Class type;           // our type
-    private final Class recordType;     // the underlying records that are produced by this track
     private final String name;          // the name
     private final File file;            // the associated file we create the reader from
 
@@ -90,7 +92,6 @@ public class RMDTrack {
      */
     public RMDTrack(Class type, String name, File file, FeatureSource reader, SAMSequenceDictionary dict, GenomeLocParser genomeLocParser, FeatureCodec codec) {
         this.type = type;
-        this.recordType = codec.getFeatureType();
         this.name = name;
         this.file = file;
         this.reader = reader;
@@ -112,19 +113,8 @@ public class RMDTrack {
     }
 
     public CloseableIterator<GATKFeature> query(GenomeLoc interval) throws IOException {
-        return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(interval.getContig(),interval.getStart(),interval.getStop()),this.getName());
+        if ( DEBUG ) logger.debug("Issuing query for %s: " + interval);
-    }
+        return new FeatureToGATKFeatureIterator(genomeLocParser, reader.query(interval.getContig(),interval.getStart(),interval.getStop()), this.getName());
-
-    public CloseableIterator<GATKFeature> query(GenomeLoc interval, boolean contained) throws IOException {
-        return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(interval.getContig(),interval.getStart(),interval.getStop()),this.getName());
-    }
-
-    public CloseableIterator<GATKFeature> query(String contig, int start, int stop) throws IOException {
-        return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(contig,start,stop),this.getName());
-    }
-
-    public CloseableIterator<GATKFeature> query(String contig, int start, int stop, boolean contained) throws IOException {
-        return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(contig,start,stop),this.getName());
     }
 
     public void close() {

public/java/src/org/broadinstitute/sting/utils/Utils.java

@@ -29,6 +29,7 @@ import net.sf.samtools.util.StringUtil;
 import org.apache.log4j.Logger;
 import org.broadinstitute.sting.utils.collections.Pair;
 
+import java.net.InetAddress;
 import java.util.*;
 
 /**
@@ -633,4 +634,20 @@ public class Utils {
     public static boolean isFlagSet(int value, int flag) {
         return ((value & flag) == flag);
     }
+
+    /**
+     * Helper utility that calls into the InetAddress system to resolve the hostname.  If this fails,
+     * unresolvable gets returned instead.
+     *
+     * @return
+     */
+    public static final String resolveHostname() {
+        try {
+            return InetAddress.getLocalHost().getCanonicalHostName();
+        }
+        catch (java.net.UnknownHostException uhe) { // [beware typo in code sample -dmw]
+            return "unresolvable";
+            // handle exception
+        }
+    }
 }

public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala

@@ -116,9 +116,11 @@ class QCommandLine extends CommandLineProgram with Logging {
     for (script <- allQScripts) {
       script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success)
       if ( ! settings.disableJobReport ) {
-        logger.info("Writing JobLogging GATKReport to file " + settings.jobReportFile)
+        val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt")
-        QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), settings.jobReportFile)
+        val jobReportFile = new File(jobStringName)
-        QJobReport.plotReport(settings.rScriptArgs, settings.jobReportFile)
+        logger.info("Writing JobLogging GATKReport to file " + jobReportFile)
+        QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), jobReportFile)
+        QJobReport.plotReport(settings.rScriptArgs, jobReportFile)
       }
     }
 

public/scala/src/org/broadinstitute/sting/queue/QScript.scala

@@ -64,7 +64,8 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon
    */
   def onExecutionDone(jobs: Map[QFunction, JobRunInfo], success: Boolean) {
     logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", jobs.size))
-    for ( (f, info) <- jobs ) logger.info("  %s %s".format(f.jobName, info))
+    // this is too much output
+    // for ( (f, info) <- jobs ) logger.info("  %s %s".format(f.jobName, info))
   }
 
   /**

public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala

@@ -3,6 +3,7 @@ package org.broadinstitute.sting.queue.engine
 import org.broadinstitute.sting.queue.function.InProcessFunction
 import java.util.Date
 import org.broadinstitute.sting.queue.util.{Logging, IOUtils}
+import org.broadinstitute.sting.utils.Utils
 
 /**
  * Runs a function that executes in process and does not fork out an external process.
@@ -12,6 +13,7 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce
 
   def start() = {
     getRunInfo.startTime = new Date()
+    getRunInfo.exechosts = Utils.resolveHostname()
     runStatus = RunnerStatus.RUNNING
 
     function.run()

public/scala/src/org/broadinstitute/sting/queue/engine/JobRunInfo.scala

@@ -39,6 +39,7 @@ class JobRunInfo {
   var startTime: Date = _
   /** The done time with millisecond resolution of this job */
   var doneTime: Date = _
+  var exechosts: String = "localhost"
 
   def getStartTime = startTime
   def getDoneTime = doneTime
@@ -48,6 +49,8 @@ class JobRunInfo {
   /** Helper function that pretty prints the date */
   private def formatTime(d: Date) = if ( d != null ) formatter.format(d) else "null"
 
+  def getExecHosts = exechosts
+
   /**
    * Was any information set for this jobInfo?  JobInfo can be unset because
    * the job never ran or because it already completed.

public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala

@@ -39,6 +39,7 @@ import org.broadinstitute.sting.queue.util._
 import collection.immutable.{TreeSet, TreeMap}
 import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction}
 import java.util.Date
+import org.broadinstitute.sting.utils.Utils
 
 /**
  * The internal dependency tracker between sets of function input and output files.
@@ -321,6 +322,7 @@ class QGraph extends Logging {
       foreachFunction(readyJobs.toList, edge => {
         if (running) {
           edge.myRunInfo.startTime = new Date()
+          edge.getRunInfo.exechosts = Utils.resolveHostname()
           logEdge(edge)
           edge.myRunInfo.doneTime = new Date()
           edge.markAsDone

public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala

@@ -71,7 +71,7 @@ class QGraphSettings {
   var expandedDotFile: File = _
 
   @Argument(fullName="jobReport", shortName="jobReport", doc="File where we will write the Queue job report", required=false)
-  var jobReportFile: File = new File("queue_jobreport.gatkreport.txt")
+  var jobReportFile: String = _
 
   @Advanced
   @Argument(fullName="disableJobReport", shortName="disabpleJobReport", doc="If provided, we will not create a job report", required=false)

public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala

@@ -276,6 +276,13 @@ object Lsf706JobRunner extends Logging {
       // the platform LSF startTimes are in seconds, not milliseconds, so convert to the java convention
       runner.getRunInfo.startTime = new Date(jobInfo.startTime.longValue * 1000)
       runner.getRunInfo.doneTime = new Date(jobInfo.endTime.longValue * 1000)
+      val exHostsRaw = jobInfo.exHosts.getStringArray(0)
+      //logger.warn("exHostsRaw = " + exHostsRaw)
+      val exHostsList = exHostsRaw.toList
+      //logger.warn("exHostsList = " + exHostsList)
+      val exHosts = exHostsList.reduceLeft(_ + "," + _)
+      //logger.warn("exHosts = " + exHosts)
+      runner.getRunInfo.exechosts = exHosts
     }
 
     runner.updateStatus(

public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala

@@ -28,6 +28,8 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction
 import org.broadinstitute.sting.queue.util.ShellJob
 import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner}
 import java.util.Date
+import org.broadinstitute.sting.gatk.phonehome.GATKRunReport
+import org.broadinstitute.sting.utils.Utils
 
 /**
  * Runs jobs one at a time locally
@@ -52,6 +54,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRu
     updateJobRun(job)
 
     getRunInfo.startTime = new Date()
+    getRunInfo.exechosts = Utils.resolveHostname()
     updateStatus(RunnerStatus.RUNNING)
     job.run()
     getRunInfo.doneTime = new Date()

public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala

@@ -48,12 +48,13 @@ trait QJobReport extends Logging {
   def disableReport() { reportEnabled = false }
 
   def setRunInfo(info: JobRunInfo) {
-    logger.info("info " + info)
+    //logger.info("info " + info)
     reportFeatures = Map(
       "iteration" -> 1,
       "analysisName" -> getReportGroup,
       "jobName" -> QJobReport.workAroundSameJobNames(this),
       "intermediate" -> self.isIntermediate,
+      "exechosts" -> info.getExecHosts,
       "startTime" -> info.getStartTime.getTime,
       "doneTime" -> info.getDoneTime.getTime,
       "formattedStartTime" -> info.getFormattedStartTime,

public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala

@@ -57,4 +57,6 @@ object QScriptUtils {
   }
 
 
+  def ?[A <: AnyRef](ref: A): Option[A] =
+    if (ref eq null) None else Some(ref)
 }