diff --git a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java index acee1a6a3..4d94130a8 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java +++ b/public/java/src/org/broadinstitute/sting/gatk/phonehome/GATKRunReport.java @@ -46,7 +46,6 @@ import org.simpleframework.xml.stream.Format; import org.simpleframework.xml.stream.HyphenStyle; import java.io.*; -import java.net.InetAddress; import java.security.NoSuchAlgorithmException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -230,22 +229,6 @@ public class GATKRunReport { } - /** - * Helper utility that calls into the InetAddress system to resolve the hostname. If this fails, - * unresolvable gets returned instead. - * - * @return - */ - private String resolveHostname() { - try { - return InetAddress.getLocalHost().getCanonicalHostName(); - } - catch (java.net.UnknownHostException uhe) { // [beware typo in code sample -dmw] - return "unresolvable"; - // handle exception - } - } - public void postReport(PhoneHomeOption type) { logger.debug("Posting report of type " + type); switch (type) { @@ -325,7 +308,7 @@ public class GATKRunReport { private void postReportToAWSS3() { // modifying example code from http://jets3t.s3.amazonaws.com/toolkit/code-samples.html - this.hostName = resolveHostname(); // we want to fill in the host name + this.hostName = Utils.resolveHostname(); // we want to fill in the host name File localFile = postReportToLocalDisk(new File("./")); logger.debug("Generating GATK report to AWS S3 based on local file " + localFile); if ( localFile != null ) { // we succeeded in creating the local file diff --git a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java index ba1ca674e..a97f3211c 100644 --- a/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java +++ b/public/java/src/org/broadinstitute/sting/gatk/refdata/tracks/RMDTrack.java @@ -25,6 +25,7 @@ package org.broadinstitute.sting.gatk.refdata.tracks; import net.sf.samtools.SAMSequenceDictionary; import net.sf.samtools.util.CloseableIterator; +import org.apache.log4j.Logger; import org.broad.tribble.FeatureCodec; import org.broad.tribble.FeatureSource; import org.broadinstitute.sting.gatk.refdata.utils.FeatureToGATKFeatureIterator; @@ -45,10 +46,11 @@ import java.io.IOException; * the basics of what a reference metadata track must contain. */ public class RMDTrack { + private final static Logger logger = Logger.getLogger(RMDTrackBuilder.class); + private final static boolean DEBUG = false; // the basics of a track: private final Class type; // our type - private final Class recordType; // the underlying records that are produced by this track private final String name; // the name private final File file; // the associated file we create the reader from @@ -90,7 +92,6 @@ public class RMDTrack { */ public RMDTrack(Class type, String name, File file, FeatureSource reader, SAMSequenceDictionary dict, GenomeLocParser genomeLocParser, FeatureCodec codec) { this.type = type; - this.recordType = codec.getFeatureType(); this.name = name; this.file = file; this.reader = reader; @@ -112,19 +113,8 @@ public class RMDTrack { } public CloseableIterator query(GenomeLoc interval) throws IOException { - return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(interval.getContig(),interval.getStart(),interval.getStop()),this.getName()); - } - - public CloseableIterator query(GenomeLoc interval, boolean contained) throws IOException { - return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(interval.getContig(),interval.getStart(),interval.getStop()),this.getName()); - } - - public CloseableIterator query(String contig, int start, int stop) throws IOException { - return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(contig,start,stop),this.getName()); - } - - public CloseableIterator query(String contig, int start, int stop, boolean contained) throws IOException { - return new FeatureToGATKFeatureIterator(genomeLocParser,reader.query(contig,start,stop),this.getName()); + if ( DEBUG ) logger.debug("Issuing query for %s: " + interval); + return new FeatureToGATKFeatureIterator(genomeLocParser, reader.query(interval.getContig(),interval.getStart(),interval.getStop()), this.getName()); } public void close() { diff --git a/public/java/src/org/broadinstitute/sting/utils/Utils.java b/public/java/src/org/broadinstitute/sting/utils/Utils.java index 015e5d6f6..f6edb319f 100755 --- a/public/java/src/org/broadinstitute/sting/utils/Utils.java +++ b/public/java/src/org/broadinstitute/sting/utils/Utils.java @@ -29,6 +29,7 @@ import net.sf.samtools.util.StringUtil; import org.apache.log4j.Logger; import org.broadinstitute.sting.utils.collections.Pair; +import java.net.InetAddress; import java.util.*; /** @@ -633,4 +634,20 @@ public class Utils { public static boolean isFlagSet(int value, int flag) { return ((value & flag) == flag); } + + /** + * Helper utility that calls into the InetAddress system to resolve the hostname. If this fails, + * unresolvable gets returned instead. + * + * @return + */ + public static final String resolveHostname() { + try { + return InetAddress.getLocalHost().getCanonicalHostName(); + } + catch (java.net.UnknownHostException uhe) { // [beware typo in code sample -dmw] + return "unresolvable"; + // handle exception + } + } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala index 138003cdd..297da8cc9 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QCommandLine.scala @@ -116,9 +116,11 @@ class QCommandLine extends CommandLineProgram with Logging { for (script <- allQScripts) { script.onExecutionDone(qGraph.getFunctionsAndStatus(script.functions), qGraph.success) if ( ! settings.disableJobReport ) { - logger.info("Writing JobLogging GATKReport to file " + settings.jobReportFile) - QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), settings.jobReportFile) - QJobReport.plotReport(settings.rScriptArgs, settings.jobReportFile) + val jobStringName = (QScriptUtils.?(settings.jobReportFile)).getOrElse(settings.qSettings.jobNamePrefix + ".jobreport.txt") + val jobReportFile = new File(jobStringName) + logger.info("Writing JobLogging GATKReport to file " + jobReportFile) + QJobReport.printReport(qGraph.getFunctionsAndStatus(script.functions), jobReportFile) + QJobReport.plotReport(settings.rScriptArgs, jobReportFile) } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala index 3120d5d62..fce65c997 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/QScript.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/QScript.scala @@ -64,7 +64,8 @@ trait QScript extends Logging with PrimitiveOptionConversions with StringFileCon */ def onExecutionDone(jobs: Map[QFunction, JobRunInfo], success: Boolean) { logger.info("Script %s with %d total jobs".format(if (success) "completed successfully" else "failed", jobs.size)) - for ( (f, info) <- jobs ) logger.info(" %s %s".format(f.jobName, info)) + // this is too much output + // for ( (f, info) <- jobs ) logger.info(" %s %s".format(f.jobName, info)) } /** diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala index 25baaca4f..a580be473 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/InProcessRunner.scala @@ -3,6 +3,7 @@ package org.broadinstitute.sting.queue.engine import org.broadinstitute.sting.queue.function.InProcessFunction import java.util.Date import org.broadinstitute.sting.queue.util.{Logging, IOUtils} +import org.broadinstitute.sting.utils.Utils /** * Runs a function that executes in process and does not fork out an external process. @@ -12,6 +13,7 @@ class InProcessRunner(val function: InProcessFunction) extends JobRunner[InProce def start() = { getRunInfo.startTime = new Date() + getRunInfo.exechosts = Utils.resolveHostname() runStatus = RunnerStatus.RUNNING function.run() diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/JobRunInfo.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/JobRunInfo.scala index 03124a420..2caa4d2aa 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/JobRunInfo.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/JobRunInfo.scala @@ -39,6 +39,7 @@ class JobRunInfo { var startTime: Date = _ /** The done time with millisecond resolution of this job */ var doneTime: Date = _ + var exechosts: String = "localhost" def getStartTime = startTime def getDoneTime = doneTime @@ -48,6 +49,8 @@ class JobRunInfo { /** Helper function that pretty prints the date */ private def formatTime(d: Date) = if ( d != null ) formatter.format(d) else "null" + def getExecHosts = exechosts + /** * Was any information set for this jobInfo? JobInfo can be unset because * the job never ran or because it already completed. diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala index 557230b05..4469874e2 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraph.scala @@ -39,6 +39,7 @@ import org.broadinstitute.sting.queue.util._ import collection.immutable.{TreeSet, TreeMap} import org.broadinstitute.sting.queue.function.scattergather.{ScatterFunction, CloneFunction, GatherFunction, ScatterGatherableFunction} import java.util.Date +import org.broadinstitute.sting.utils.Utils /** * The internal dependency tracker between sets of function input and output files. @@ -321,6 +322,7 @@ class QGraph extends Logging { foreachFunction(readyJobs.toList, edge => { if (running) { edge.myRunInfo.startTime = new Date() + edge.getRunInfo.exechosts = Utils.resolveHostname() logEdge(edge) edge.myRunInfo.doneTime = new Date() edge.markAsDone diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala index 46063fc24..ee498c8a0 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/QGraphSettings.scala @@ -71,7 +71,7 @@ class QGraphSettings { var expandedDotFile: File = _ @Argument(fullName="jobReport", shortName="jobReport", doc="File where we will write the Queue job report", required=false) - var jobReportFile: File = new File("queue_jobreport.gatkreport.txt") + var jobReportFile: String = _ @Advanced @Argument(fullName="disableJobReport", shortName="disabpleJobReport", doc="If provided, we will not create a job report", required=false) diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala index 166008c26..323cc63ff 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/lsf/Lsf706JobRunner.scala @@ -276,6 +276,13 @@ object Lsf706JobRunner extends Logging { // the platform LSF startTimes are in seconds, not milliseconds, so convert to the java convention runner.getRunInfo.startTime = new Date(jobInfo.startTime.longValue * 1000) runner.getRunInfo.doneTime = new Date(jobInfo.endTime.longValue * 1000) + val exHostsRaw = jobInfo.exHosts.getStringArray(0) + //logger.warn("exHostsRaw = " + exHostsRaw) + val exHostsList = exHostsRaw.toList + //logger.warn("exHostsList = " + exHostsList) + val exHosts = exHostsList.reduceLeft(_ + "," + _) + //logger.warn("exHosts = " + exHosts) + runner.getRunInfo.exechosts = exHosts } runner.updateStatus( diff --git a/public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala b/public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala index 4124f65a0..ae899868a 100755 --- a/public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/engine/shell/ShellJobRunner.scala @@ -28,6 +28,8 @@ import org.broadinstitute.sting.queue.function.CommandLineFunction import org.broadinstitute.sting.queue.util.ShellJob import org.broadinstitute.sting.queue.engine.{RunnerStatus, CommandLineJobRunner} import java.util.Date +import org.broadinstitute.sting.gatk.phonehome.GATKRunReport +import org.broadinstitute.sting.utils.Utils /** * Runs jobs one at a time locally @@ -52,6 +54,7 @@ class ShellJobRunner(val function: CommandLineFunction) extends CommandLineJobRu updateJobRun(job) getRunInfo.startTime = new Date() + getRunInfo.exechosts = Utils.resolveHostname() updateStatus(RunnerStatus.RUNNING) job.run() getRunInfo.doneTime = new Date() diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala index 04ae54c21..85896da66 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QJobReport.scala @@ -48,12 +48,13 @@ trait QJobReport extends Logging { def disableReport() { reportEnabled = false } def setRunInfo(info: JobRunInfo) { - logger.info("info " + info) + //logger.info("info " + info) reportFeatures = Map( "iteration" -> 1, "analysisName" -> getReportGroup, "jobName" -> QJobReport.workAroundSameJobNames(this), "intermediate" -> self.isIntermediate, + "exechosts" -> info.getExecHosts, "startTime" -> info.getStartTime.getTime, "doneTime" -> info.getDoneTime.getTime, "formattedStartTime" -> info.getFormattedStartTime, diff --git a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala index 12bd880d8..3b1b2ece1 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/util/QScriptUtils.scala @@ -57,4 +57,6 @@ object QScriptUtils { } + def ?[A <: AnyRef](ref: A): Option[A] = + if (ref eq null) None else Some(ref) } \ No newline at end of file