From c18beadbdb4b864d4e04d453a6edebac7ab0818a Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 23 Jan 2012 16:17:04 -0500 Subject: [PATCH] Device files like /dev/null are now tracked as special by Queue and are not used to generate .out file paths, scattered into a temporary directory, gathered, deleted, etc. Attempted workaround for xdr_resourceInfoReq unsatisfied link during loading of libbat.so. --- .../sting/jna/lsf/v7_0_6/LibBat.java | 24 ++++--- .../gatk/ArgumentDefinitionField.java | 6 +- .../sting/utils/io/IOUtils.java | 19 +++++- .../sting/utils/io/IOUtilsUnitTest.java | 36 ++++++++++ .../qscripts/examples/DevNullOutput.scala | 49 ++++++++++++++ .../sting/queue/function/QFunction.scala | 8 ++- .../ScatterGatherableFunction.scala | 10 +-- .../examples/DevNullOutputPipelineTest.scala | 67 +++++++++++++++++++ .../ExampleUnifiedGenotyperPipelineTest.scala | 1 + 9 files changed, 199 insertions(+), 21 deletions(-) create mode 100644 public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala create mode 100644 public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala diff --git a/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java b/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java index d7b34a253..f948a9bcf 100644 --- a/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java +++ b/public/java/src/org/broadinstitute/sting/jna/lsf/v7_0_6/LibBat.java @@ -71,6 +71,14 @@ import org.broadinstitute.sting.jna.clibrary.LibC; public class LibBat { static { + // via Platform LSF Configuration Reference, by default quiet the BSUB output. + if ("Y".equals(System.getProperty("BSUB_QUIET", "Y"))) + LibC.setenv("BSUB_QUIET", "Y", 1); + String lsfLibDir = System.getenv("LSF_LIBDIR"); + if (lsfLibDir != null) { + NativeLibrary.addSearchPath("lsf", lsfLibDir); + NativeLibrary.addSearchPath("bat", lsfLibDir); + } /* LSF 7.0.6 on the mac is missing the unsatisfied exported symbol for environ which was removed on MacOS X 10.5+. nm $LSF_LIBDIR/liblsf.dylib | grep environ @@ -79,16 +87,14 @@ public class LibBat { */ if (Platform.isMac()) NativeLibrary.getInstance("environhack"); - String lsfLibDir = System.getenv("LSF_LIBDIR"); - if (lsfLibDir != null) { - NativeLibrary.addSearchPath("lsf", lsfLibDir); - NativeLibrary.addSearchPath("bat", lsfLibDir); - } - NativeLibrary.getInstance("lsf"); - // via Platform LSF Configuration Reference, by default quiet the BSUB output. - if ("Y".equals(System.getProperty("BSUB_QUIET", "Y"))) - LibC.setenv("BSUB_QUIET", "Y", 1); + NativeLibrary liblsf = NativeLibrary.getInstance("lsf"); Native.register("bat"); + // HACK: Running into a weird error: + // java.lang.UnsatisfiedLinkError: Unable to load library 'bat': <$LSF_LIBDIR>/libbat.so: undefined symbol: xdr_resourceInfoReq + // This function is very clearly unsatisfied by running 'nm $LSF_LIBDIR/libbat.so | grep xdr_resourceInfoReq' but is + // found in liblsf.so when running 'nm $LSF_LIBDIR/liblsf.so | grep xdr_resourceInfoReq'. For now holding on to a reference + // to the LSF lib just in case this is a problem with the NativeLibrary's internal WeakReferences and the library being unloaded? + liblsf.getFunction("xdr_resourceInfoReq").getName(); } // Via support@platform.com: diff --git a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java index 71640c66a..00a6ac1ae 100644 --- a/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java +++ b/public/java/src/org/broadinstitute/sting/queue/extensions/gatk/ArgumentDefinitionField.java @@ -468,7 +468,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (!org.broadinstitute.sting.gatk.io.stubs.VCFWriterArgumentTypeDescriptor.isCompressed(%2$s.getPath))%n" + " %1$s = new File(%2$s.getPath + \"%3$s\")%n"), auxFieldName, originalFieldName, Tribble.STANDARD_INDEX_EXTENSION); @@ -481,7 +481,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (!%3$s)%n" + " %1$s = new File(%2$s.getPath.stripSuffix(\".bam\") + \"%4$s\")%n"), auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.DISABLE_INDEXING_FULLNAME, BAMIndex.BAMIndexSuffix); @@ -494,7 +494,7 @@ public abstract class ArgumentDefinitionField extends ArgumentField { } @Override protected String getFreezeFields() { return String.format( - ("if (%2$s != null)%n" + + ("if (%2$s != null && !org.broadinstitute.sting.utils.io.IOUtils.isSpecialFile(%2$s))%n" + " if (%3$s)%n" + " %1$s = new File(%2$s.getPath + \"%4$s\")%n"), auxFieldName, originalFieldName, SAMFileWriterArgumentTypeDescriptor.ENABLE_MD5_FULLNAME, ".md5"); diff --git a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java index b3fdb93d3..a5ba857ef 100644 --- a/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java +++ b/public/java/src/org/broadinstitute/sting/utils/io/IOUtils.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, The Broad Institute + * Copyright (c) 2012, The Broad Institute * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation @@ -37,6 +37,7 @@ import java.util.*; public class IOUtils { private static Logger logger = Logger.getLogger(IOUtils.class); + private static final File DEV_DIR = new File("/dev"); /** * Checks if the temp directory has been setup and throws an exception if they user hasn't set it correctly. @@ -301,12 +302,17 @@ public class IOUtils { } /** - * Tries to delete a file. Emits a warning if the file was unable to be deleted. + * Tries to delete a file. Emits a warning if the file + * is not a special file and was unable to be deleted. * * @param file File to delete. * @return true if the file was deleted. */ public static boolean tryDelete(File file) { + if (isSpecialFile(file)) { + logger.debug("Not trying to delete " + file); + return false; + } boolean deleted = FileUtils.deleteQuietly(file); if (deleted) logger.debug("Deleted " + file); @@ -385,4 +391,13 @@ public class IOUtils { } } + + /** + * Returns true if the file is a special file. + * @param file File path to check. + * @return true if the file is a special file. + */ + public static boolean isSpecialFile(File file) { + return file != null && (file.getAbsolutePath().startsWith("/dev/") || file.equals(DEV_DIR)); + } } diff --git a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java index 4caf7f485..757e6efdf 100644 --- a/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java +++ b/public/java/test/org/broadinstitute/sting/utils/io/IOUtilsUnitTest.java @@ -1,3 +1,27 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + package org.broadinstitute.sting.utils.io; import org.apache.commons.io.FileUtils; @@ -194,4 +218,16 @@ public class IOUtilsUnitTest extends BaseTest { Assert.assertEquals(resource.getPath(), "foo"); Assert.assertEquals(resource.getRelativeClass(), Resource.class); } + + @Test + public void testIsSpecialFile() { + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/null"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/full"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stdout"))); + Assert.assertTrue(IOUtils.isSpecialFile(new File("/dev/stderr"))); + Assert.assertFalse(IOUtils.isSpecialFile(null)); + Assert.assertFalse(IOUtils.isSpecialFile(new File("/home/user/my.file"))); + Assert.assertFalse(IOUtils.isSpecialFile(new File("/devfake/null"))); + } } diff --git a/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala new file mode 100644 index 000000000..d891ebaaf --- /dev/null +++ b/public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.qscripts.examples + +import org.broadinstitute.sting.queue.QScript +import org.broadinstitute.sting.queue.extensions.gatk._ + +/** + * Script used for testing output to /dev/null + */ +class DevNullOutput extends QScript { + @Input(doc="The reference file for the bam files.", shortName="R") + var referenceFile: File = _ + + @Input(doc="Bam file to genotype.", shortName="I") + var bamFile: File = _ + + def script() { + val genotyper = new UnifiedGenotyper + genotyper.reference_sequence = referenceFile + genotyper.memoryLimit = 2 + genotyper.scatterCount = 3 + genotyper.input_file :+= bamFile + genotyper.out = "/dev/null" + add(genotyper) + } +} diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala index dee1acfac..7d9debbdc 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/QFunction.scala @@ -163,7 +163,9 @@ trait QFunction extends Logging with QJobReport { * Returns prefixes for hidden done/fail files. * @return prefixes. */ - private def statusPrefixes = statusPaths.map(file => file.getParentFile + "/." + file.getName) + private def statusPrefixes = statusPaths. + filter(file => !IOUtils.isSpecialFile(file)). + map(file => file.getParentFile + "/." + file.getName) /** * Returns the output files for this function. @@ -236,7 +238,7 @@ trait QFunction extends Logging with QJobReport { * Deletes the output files and all the status files for this function. */ def deleteOutputs() { - outputs.foreach(file => IOUtils.tryDelete(file)) + outputs.filter(file => !IOUtils.isSpecialFile(file)).foreach(file => IOUtils.tryDelete(file)) doneOutputs.foreach(file => IOUtils.tryDelete(file)) failOutputs.foreach(file => IOUtils.tryDelete(file)) } @@ -346,7 +348,7 @@ trait QFunction extends Logging with QJobReport { if (jobOutputFile == null) { jobOutputFile = firstOutput match { - case file: File => new File(file.getParentFile, file.getName + ".out") + case file: File if (!IOUtils.isSpecialFile(file)) => new File(file.getParentFile, file.getName + ".out") case _ => new File(jobName + ".out") } } diff --git a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala index 921928bce..4578f0e82 100644 --- a/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala +++ b/public/scala/src/org/broadinstitute/sting/queue/function/scattergather/ScatterGatherableFunction.scala @@ -134,8 +134,10 @@ trait ScatterGatherableFunction extends CommandLineFunction { var gatherOutputs = ListMap.empty[ArgumentSource, File] var gatherAddOrder = numClones + 2 - // Only track fields that will have a value - val outputFieldsWithValues = this.outputFields.filter(hasFieldValue(_)) + // Only track fields that will have an output file + val outputFieldsWithValues = this.outputFields. + filter(hasFieldValue(_)). + filter(gatherField => !IOUtils.isSpecialFile(getFieldFile(gatherField))) for (gatherField <- outputFieldsWithValues) { gatherOutputs += gatherField -> getFieldFile(gatherField) @@ -175,9 +177,9 @@ trait ScatterGatherableFunction extends CommandLineFunction { cloneFunction.analysisName = this.analysisName cloneFunction.cloneIndex = i cloneFunction.commandDirectory = this.scatterGatherTempDir(dirFormat.format(i)) - cloneFunction.jobOutputFile = new File(this.jobOutputFile.getName) + cloneFunction.jobOutputFile = if (IOUtils.isSpecialFile(this.jobOutputFile)) this.jobOutputFile else new File(this.jobOutputFile.getName) if (this.jobErrorFile != null) - cloneFunction.jobErrorFile = new File(this.jobErrorFile.getName) + cloneFunction.jobErrorFile = if (IOUtils.isSpecialFile(this.jobErrorFile)) this.jobErrorFile else new File(this.jobErrorFile.getName) cloneFunction.addOrder = this.addOrder :+ (i+1) cloneFunction.isIntermediate = true diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala new file mode 100644 index 000000000..9bb287ac4 --- /dev/null +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/DevNullOutputPipelineTest.scala @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2012, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +package org.broadinstitute.sting.queue.pipeline.examples + +/* + * Copyright (c) 2011, The Broad Institute + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +import org.testng.annotations.Test +import org.broadinstitute.sting.queue.pipeline.{PipelineTest, PipelineTestSpec} +import org.broadinstitute.sting.BaseTest + +class DevNullOutputPipelineTest { + @Test + def testDevNullOutput() { + val spec = new PipelineTestSpec + spec.name = "devnulloutput" + spec.args = Array( + " -S public/scala/qscript/org/broadinstitute/sting/queue/qscripts/examples/DevNullOutput.scala", + " -R " + BaseTest.testDir + "exampleFASTA.fasta", + " -I " + BaseTest.testDir + "exampleBAM.bam").mkString + spec.jobRunners = PipelineTest.allJobRunners + PipelineTest.executeTest(spec) + } +} diff --git a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala index d50673a1a..f598402af 100644 --- a/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala +++ b/public/scala/test/org/broadinstitute/sting/queue/pipeline/examples/ExampleUnifiedGenotyperPipelineTest.scala @@ -39,6 +39,7 @@ class ExampleUnifiedGenotyperPipelineTest { " -I " + BaseTest.testDir + "exampleBAM.bam", " -filter QD", " -filterExpression 'QD < 2.0'").mkString + spec.jobRunners = PipelineTest.allJobRunners PipelineTest.executeTest(spec) } }