Device files like /dev/null are now tracked as special by Queue and are not used to generate .out file paths, scattered into a temporary directory, gathered, deleted, etc.

Attempted workaround for xdr_resourceInfoReq unsatisfied link during loading of libbat.so.

public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java

@@ -71,6 +71,14 @@ import org.broadinstitute.sting.jna.clibrary.LibC;
 public class LibBat {
 
     static {
+        // via Platform LSF Configuration Reference, by default quiet the BSUB output.
+        if ("Y".equals(System.getProperty("BSUB_QUIET", "Y")))
+            LibC.setenv("BSUB_QUIET", "Y", 1);
+        String lsfLibDir = System.getenv("LSF_LIBDIR");
+        if (lsfLibDir != null) {
+            NativeLibrary.addSearchPath("lsf", lsfLibDir);
+            NativeLibrary.addSearchPath("bat", lsfLibDir);
+        }
         /*
         LSF 7.0.6 on the mac is missing the unsatisfied exported symbol for environ which was removed on MacOS X 10.5+.
         nm $LSF_LIBDIR/liblsf.dylib | grep environ
@@ -79,16 +87,14 @@ public class LibBat {
         */
         if (Platform.isMac())
             NativeLibrary.getInstance("environhack");
-        String lsfLibDir = System.getenv("LSF_LIBDIR");
+        NativeLibrary liblsf = NativeLibrary.getInstance("lsf");
-        if (lsfLibDir != null) {
-            NativeLibrary.addSearchPath("lsf", lsfLibDir);
-            NativeLibrary.addSearchPath("bat", lsfLibDir);
-        }
-        NativeLibrary.getInstance("lsf");
-        // via Platform LSF Configuration Reference, by default quiet the BSUB output.
-        if ("Y".equals(System.getProperty("BSUB_QUIET", "Y")))
-            LibC.setenv("BSUB_QUIET", "Y", 1);
         Native.register("bat");
+        // HACK: Running into a weird error:
+        //   java.lang.UnsatisfiedLinkError: Unable to load library 'bat': <$LSF_LIBDIR>/libbat.so: undefined symbol: xdr_resourceInfoReq
+        // This function is very clearly unsatisfied by running 'nm $LSF_LIBDIR/libbat.so | grep xdr_resourceInfoReq' but is
+        // found in liblsf.so when running 'nm $LSF_LIBDIR/liblsf.so | grep xdr_resourceInfoReq'. For now holding on to a reference
+        // to the LSF lib just in case this is a problem with the NativeLibrary's internal WeakReferences and the library being unloaded?
+        liblsf.getFunction("xdr_resourceInfoReq").getName();
     }
 
     // Via support@platform.com:

public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java

@@ -468,7 +468,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
         }
         @Override protected String getFreezeFields() {
             return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                             "  if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" +
                             "    %1$s = new File(%2$s.getPath + \"%3$s\")%n"),
                     auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION);
@@ -481,7 +481,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
         }
         @Override protected String getFreezeFields() {
             return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                             "  if (!%3$s)%n" +
                             "    %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"),
                     auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix);
@@ -494,7 +494,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField {
         }
         @Override protected String getFreezeFields() {
             return String.format(
-                    ("if (%2$s != null)%n" +
+                    ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" +
                             "  if (%3$s)%n" +
                             "    %1$s = new File(%2$s.getPath + \"%4$s\")%n"),
                     auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5");

public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2011, The Broad Institute
+ * Copyright (c) 2012, The Broad Institute
  *
  * Permission is hereby granted, free of charge, to any person
  * obtaining a copy of this software and associated documentation
@@ -37,6 +37,7 @@ import java.util.*;
 
 public class IOUtils {
     private static Logger logger = Logger.getLogger(IOUtils.class);
+    private static final File DEV_DIR = new File("/dev");
 
     /**
      * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly.
@@ -301,12 +302,17 @@ public class IOUtils {
     }
 
     /**
-     * Tries to delete a file. Emits a warning if the file was unable to be deleted.
+     * Tries to delete a file. Emits a warning if the file
+     * is not a special file and was unable to be deleted.
      *
      * @param file File to delete.
      * @return true if the file was deleted.
      */
     public static boolean tryDelete(File file) {
+        if (isSpecialFile(file)) {
+            logger.debug("Not trying to delete " + file);
+            return false;
+        }
         boolean deleted = FileUtils.deleteQuietly(file);
         if (deleted)
             logger.debug("Deleted " + file);
@@ -385,4 +391,13 @@ public class IOUtils {
         }
 
     }
+
+    /**
+     * Returns true if the file is a special file.
+     * @param file File path to check.
+     * @return true if the file is a special file.
+     */
+    public static boolean isSpecialFile(File file) {
+        return file != null && (file.getAbsolutePath().startsWith("/dev/") || file.equals(DEV_DIR));
+    }
 }

public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java

@@ -1,3 +1,27 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
 package org.broadinstitute.sting.utils.io;
 
 import org.apache.commons.io.FileUtils;
@@ -194,4 +218,16 @@ public class IOUtilsUnitTest extends BaseTest {
         Assert.assertEquals(resource.getPath(), "foo");
         Assert.assertEquals(resource.getRelativeClass(), Resource.class);
     }
+
+    @Test
+    public void testIsSpecialFile() {
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/null")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/full")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stdout")));
+        Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stderr")));
+        Assert.assertFalse(IOUtils.isSpecialFile(null));
+        Assert.assertFalse(IOUtils.isSpecialFile(new File("/home/user/my.file")));
+        Assert.assertFalse(IOUtils.isSpecialFile(new File("/devfake/null")));
+    }
 }

public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.queue.qscripts.examples
+
+import org.broadinstitute.sting.queue.QScript
+import org.broadinstitute.sting.queue.extensions.gatk._
+
+/**
+ * Script used for testing output to /dev/null
+ */
+class DevNullOutput extends QScript {
+  @Input(doc="The reference file for the bam files.", shortName="R")
+  var referenceFile: File = _
+
+  @Input(doc="Bam file to genotype.", shortName="I")
+  var bamFile: File = _
+
+  def script() {
+    val genotyper = new UnifiedGenotyper
+    genotyper.reference_sequence = referenceFile
+    genotyper.memoryLimit = 2
+    genotyper.scatterCount = 3
+    genotyper.input_file :+= bamFile
+    genotyper.out = "/dev/null"
+    add(genotyper)
+  }
+}

public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala

@@ -163,7 +163,9 @@ trait QFunction extends Logging with QJobReport {
    * Returns prefixes for hidden done/fail files.
    * @return prefixes.
    */
-  private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName)
+  private def statusPrefixes = statusPaths.
+    filter(file => !IOUtils.isSpecialFile(file)).
+    map(file => file.getParentFile + "/." + file.getName)
 
   /**
    * Returns the output files for this function.
@@ -236,7 +238,7 @@ trait QFunction extends Logging with QJobReport {
    * Deletes the output files and all the status files for this function.
    */
   def deleteOutputs() {
-    outputs.foreach(file => IOUtils.tryDelete(file))
+    outputs.filter(file => !IOUtils.isSpecialFile(file)).foreach(file => IOUtils.tryDelete(file))
     doneOutputs.foreach(file => IOUtils.tryDelete(file))
     failOutputs.foreach(file => IOUtils.tryDelete(file))
   }
@@ -346,7 +348,7 @@ trait QFunction extends Logging with QJobReport {
 
     if (jobOutputFile == null) {
       jobOutputFile = firstOutput match {
-        case file: File => new File(file.getParentFile, file.getName + ".out")
+        case file: File if (!IOUtils.isSpecialFile(file)) => new File(file.getParentFile, file.getName + ".out")
         case _ => new File(jobName + ".out")
       }
     }

public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala

@@ -134,8 +134,10 @@ trait ScatterGatherableFunction extends CommandLineFunction {
     var gatherOutputs = ListMap.empty[ArgumentSource, File]
     var gatherAddOrder = numClones + 2
 
-    // Only track fields that will have a value
+    // Only track fields that will have an output file
-    val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_))
+    val outputFieldsWithValues = this.outputFields.
+      filter(hasFieldValue(_)).
+      filter(gatherField => !IOUtils.isSpecialFile(getFieldFile(gatherField)))
 
     for (gatherField <- outputFieldsWithValues) {
       gatherOutputs += gatherField -> getFieldFile(gatherField)
@@ -175,9 +177,9 @@ trait ScatterGatherableFunction extends CommandLineFunction {
       cloneFunction.analysisName = this.analysisName
       cloneFunction.cloneIndex = i
       cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i))
-      cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName)
+      cloneFunction.jobOutputFile = if (IOUtils.isSpecialFile(this.jobOutputFile)) this.jobOutputFile else new File(this.jobOutputFile.getName)
       if (this.jobErrorFile != null)
-        cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName)
+        cloneFunction.jobErrorFile = if (IOUtils.isSpecialFile(this.jobErrorFile)) this.jobErrorFile else new File(this.jobErrorFile.getName)
       cloneFunction.addOrder = this.addOrder :+ (i+1)
       cloneFunction.isIntermediate = true
 

public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2012, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+package org.broadinstitute.sting.queue.pipeline.examples
+
+/*
+ * Copyright (c) 2011, The Broad Institute
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+import org.testng.annotations.Test
+import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec}
+import org.broadinstitute.sting.BaseTest
+
+class DevNullOutputPipelineTest {
+  @Test
+  def testDevNullOutput() {
+    val spec = new PipelineTestSpec
+    spec.name = "devnulloutput"
+    spec.args = Array(
+      " -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala",
+      " -R " + BaseTest.testDir + "exampleFASTA.fasta",
+      " -I " + BaseTest.testDir + "exampleBAM.bam").mkString
+    spec.jobRunners = PipelineTest.allJobRunners
+    PipelineTest.executeTest(spec)
+  }
+}

public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala

@@ -39,6 +39,7 @@ class ExampleUnifiedGenotyperPipelineTest {
       " -I " + BaseTest.testDir + "exampleBAM.bam",
       " -filter QD",
       " -filterExpression 'QD < 2.0'").mkString
+    spec.jobRunners = PipelineTest.allJobRunners
     PipelineTest.executeTest(spec)
   }
 }