Device files like /dev/null are now tracked as special by Queue and are not used to generate .out file paths, scattered into a temporary directory, gathered, deleted, etc.

Attempted workaround for xdr_resourceInfoReq unsatisfied link during loading of libbat.so.
2012-01-23 16:17:04 -05:00 · 2012-01-23 16:17:04 -05:00 · c18beadbdb
parent cc4ba7372f
commit c18beadbdb
9 changed files with 199 additions and 21 deletions
--- a/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java
+++ b/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java
@ -71,6 +71,14 @@ import org.broadinstitute.sting.jna.clibrary.LibC;
 public class LibBat {

    static {
+        // via Platform LSF Configuration Reference, by default quiet the BSUB output.
+        if ("Y".equals(System.getProperty("BSUB_QUIET", "Y")))
+            LibC.setenv("BSUB_QUIET", "Y", 1);
+        String lsfLibDir = System.getenv("LSF_LIBDIR");
+        if (lsfLibDir != null) {
+            NativeLibrary.addSearchPath("lsf", lsfLibDir);
+            NativeLibrary.addSearchPath("bat", lsfLibDir);
+        }
        /*
        LSF 7.0.6 on the mac is missing the unsatisfied exported symbol for environ which was removed on MacOS X 10.5+.
        nm $LSF_LIBDIR/liblsf.dylib | grep environ
@ -79,16 +87,14 @@ public class LibBat {
        */
        if (Platform.isMac())
            NativeLibrary.getInstance("environhack");
-        String lsfLibDir = System.getenv("LSF_LIBDIR");
-        if (lsfLibDir != null) {
-            NativeLibrary.addSearchPath("lsf", lsfLibDir);
-            NativeLibrary.addSearchPath("bat", lsfLibDir);
-        }
-        NativeLibrary.getInstance("lsf");
-        // via Platform LSF Configuration Reference, by default quiet the BSUB output.
-        if ("Y".equals(System.getProperty("BSUB_QUIET", "Y")))
-            LibC.setenv("BSUB_QUIET", "Y", 1);
+        NativeLibrary liblsf = NativeLibrary.getInstance("lsf");
        Native.register("bat");
+        // HACK: Running into a weird error:
+        //   java.lang.UnsatisfiedLinkError: Unable to load library 'bat': <$LSF_LIBDIR>/libbat.so: undefined symbol: xdr_resourceInfoReq
+        // This function is very clearly unsatisfied by running 'nm $LSF_LIBDIR/libbat.so | grep xdr_resourceInfoReq' but is
+        // found in liblsf.so when running 'nm $LSF_LIBDIR/liblsf.so | grep xdr_resourceInfoReq'. For now holding on to a reference
+        // to the LSF lib just in case this is a problem with the NativeLibrary's internal WeakReferences and the library being unloaded?
+        liblsf.getFunction("xdr_resourceInfoReq").getName();
    }

    // Via support@platform.com:
--- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
+++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java
@ -468,7 +468,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
        }
        @Override protected String getFreezeFields() {
            return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                            "  if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" +
                            "    %1$s = new File(%2$s.getPath + \"%3$s\")%n"),
                    auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION);
@ -481,7 +481,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
        }
        @Override protected String getFreezeFields() {
            return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                            "  if (!%3$s)%n" +
                            "    %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"),
                    auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix);
@ -494,7 +494,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
        }
        @Override protected String getFreezeFields() {
            return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                            "  if (%3$s)%n" +
                            "    %1$s = new File(%2$s.getPath + \"%4$s\")%n"),
                    auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5");
--- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java
+++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java
@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, The Broad Institute
+ * Copyright (c) 2012, The Broad Institute
 *
 * Permission is hereby granted, free of charge, to any person
 * obtaining a copy of this software and associated documentation
@ -37,6 +37,7 @@ import java.util.*;

 public class IOUtils {
    private static Logger logger = Logger.getLogger(IOUtils.class);
+    private static final File DEV_DIR = new File("/dev");

    /**
     * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
@ -301,12 +302,17 @@ public class IOUtils {
    }

    /**
-     * Tries to delete a file. Emits a warning if the file was unable to be deleted.
+     * Tries to delete a file. Emits a warning if the file
+     * is not a special file and was unable to be deleted.
     *
     * @param file File to delete.
     * @return true if the file was deleted.
     */
    public static boolean tryDelete(File file) {
+        if (isSpecialFile(file)) {
+            logger.debug("Not trying to delete " + file);
+            return false;
+        }
        boolean deleted = FileUtils.deleteQuietly(file);
        if (deleted)
            logger.debug("Deleted " + file);
@ -385,4 +391,13 @@ public class IOUtils {
        }

    }
+
+    /**
+     * Returns true if the file is a special file.
+     * @param file File path to check.
+     * @return true if the file is a special file.
+     */
+    public static boolean isSpecialFile(File file) {
+        return file != null && (file.getAbsolutePath().startsWith("/dev/") || file.equals(DEV_DIR));
+    }
 }
--- a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java
+++ b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java
@ -1,3 +1,27 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package org.broadinstitute.sting.utils.io;

 import org.apache.commons.io.FileUtils;
@ -194,4 +218,16 @@ public class IOUtilsUnitTest extends BaseTest {
        Assert.assertEquals(resource.getPath(), "foo");
        Assert.assertEquals(resource.getRelativeClass(), Resource.class);
    }
+
+    @Test
+    public void testIsSpecialFile() {
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/null")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/full")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stdout")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stderr")));
+        Assert.assertFalse(IOUtils.isSpecialFile(null));
+        Assert.assertFalse(IOUtils.isSpecialFile(new File("/home/user/my.file")));
+        Assert.assertFalse(IOUtils.isSpecialFile(new File("/devfake/null")));
+    }
 }
--- a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala
+++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala
@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.queue.qscripts.examples
+
+import org.broadinstitute.sting.queue.QScript
+import org.broadinstitute.sting.queue.extensions.gatk._
+
+/**
+ * Script used for testing output to /dev/null
+ */
+class DevNullOutput extends QScript {
+  @Input(doc="The reference file for the bam files.", shortName="R")
+  var referenceFile: File = _
+
+  @Input(doc="Bam file to genotype.", shortName="I")
+  var bamFile: File = _
+
+  def script() {
+    val genotyper = new UnifiedGenotyper
+    genotyper.reference_sequence = referenceFile
+    genotyper.memoryLimit = 2
+    genotyper.scatterCount = 3
+    genotyper.input_file :+= bamFile
+    genotyper.out = "/dev/null"
+    add(genotyper)
+  }
+}
--- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala
@ -163,7 +163,9 @@ trait QFunction extends Logging with QJobReport {
   * Returns prefixes for hidden done/fail files.
   * @return prefixes.
   */
-  private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName)
+  private def statusPrefixes = statusPaths.
+    filter(file => !IOUtils.isSpecialFile(file)).
+    map(file => file.getParentFile + "/." + file.getName)

  /**
   * Returns the output files for this function.
@ -236,7 +238,7 @@ trait QFunction extends Logging with QJobReport {
   * Deletes the output files and all the status files for this function.
   */
  def deleteOutputs() {
-    outputs.foreach(file => IOUtils.tryDelete(file))
+    outputs.filter(file => !IOUtils.isSpecialFile(file)).foreach(file => IOUtils.tryDelete(file))
    doneOutputs.foreach(file => IOUtils.tryDelete(file))
    failOutputs.foreach(file => IOUtils.tryDelete(file))
  }
@ -346,7 +348,7 @@ trait QFunction extends Logging with QJobReport {

    if (jobOutputFile == null) {
      jobOutputFile = firstOutput match {
-        case file: File => new File(file.getParentFile, file.getName + ".out")
+        case file: File if (!IOUtils.isSpecialFile(file)) => new File(file.getParentFile, file.getName + ".out")
        case _ => new File(jobName + ".out")
      }
    }
--- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala
+++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala
@ -134,8 +134,10 @@ trait ScatterGatherableFunction extends CommandLineFunction {
    var gatherOutputs = ListMap.empty[ArgumentSource, File]
    var gatherAddOrder = numClones + 2

-    // Only track fields that will have a value
-    val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
+    // Only track fields that will have an output file
+    val outputFieldsWithValues = this.outputFields.
+      filter(hasFieldValue(_)).
+      filter(gatherField => !IOUtils.isSpecialFile(getFieldFile(gatherField)))

    for (gatherField <- outputFieldsWithValues) {
      gatherOutputs += gatherField -> getFieldFile(gatherField)
@ -175,9 +177,9 @@ trait ScatterGatherableFunction extends CommandLineFunction {
      cloneFunction.analysisName = this.analysisName
      cloneFunction.cloneIndex = i
      cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i))
-      cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName)
+      cloneFunction.jobOutputFile = if (IOUtils.isSpecialFile(this.jobOutputFile)) this.jobOutputFile else new File(this.jobOutputFile.getName)
      if (this.jobErrorFile != null)
-        cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName)
+        cloneFunction.jobErrorFile = if (IOUtils.isSpecialFile(this.jobErrorFile)) this.jobErrorFile else new File(this.jobErrorFile.getName)
      cloneFunction.addOrder = this.addOrder :+ (i+1)
      cloneFunction.isIntermediate = true

--- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala
+++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala
@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.queue.pipeline.examples
+
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import org.testng.annotations.Test
+import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
+import org.broadinstitute.sting.BaseTest
+
+class DevNullOutputPipelineTest {
+  @Test
+  def testDevNullOutput() {
+    val spec = new PipelineTestSpec
+    spec.name = "devnulloutput"
+    spec.args = Array(
+      " -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala",
+      " -R " + BaseTest.testDir + "exampleFASTA.fasta",
+      " -I " + BaseTest.testDir + "exampleBAM.bam").mkString
+    spec.jobRunners = PipelineTest.allJobRunners
+    PipelineTest.executeTest(spec)
+  }
+}
--- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala
+++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala
@ -39,6 +39,7 @@ class ExampleUnifiedGenotyperPipelineTest {
      " -I " + BaseTest.testDir + "exampleBAM.bam",
      " -filter QD",
      " -filterExpression 'QD < 2.0'").mkString
+    spec.jobRunners = PipelineTest.allJobRunners
    PipelineTest.executeTest(spec)
  }
 }