zzh
/
gatk-3.8
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Attempt to determine whether underlying filesystem supports file locking and 

disable on-the-fly dict and fai generation.


git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3938 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
hanna 2010-08-04 19:28:27 +00:00
parent 1a36cb9296
commit f13d52e427
4 changed files with 118 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceDataSource.java
View File

@ -31,6 +31,7 @@ import net.sf.picard.sam.CreateSequenceDictionary;
import net.sf.picard.reference.IndexedFastaSequenceFile;
import net.sf.picard.reference.FastaSequenceIndex;
import org.broadinstitute.sting.utils.file.FSLockWithShared;
import org.broadinstitute.sting.utils.file.FileSystemInabilityToLockException;
import java.io.File;
@ -62,7 +63,7 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
*/
if (!indexFile.exists()) {
logger.info(String.format("Index file %s does not exist. Trying to create it now.", indexFile.getAbsolutePath()));
FSLockWithShared indexLock = new FSLockWithShared(indexFile);
FSLockWithShared indexLock = new FSLockWithShared(indexFile,true);
try {
// get exclusive lock
if (!indexLock.exclusiveLock())
@ -73,7 +74,13 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
FastaSequenceIndex sequenceIndex = faiBuilder.createIndex();
FastaSequenceIndexBuilder.saveAsFaiFile(sequenceIndex, indexFile);
}
catch(FileSystemInabilityToLockException ex) {
logger.info("Unable to create write lock: " + ex.getMessage());
logger.info("Skipping index creation.");
}
catch (Exception e) {
// If lock creation succeeded, the failure must have been generating the index.
// If lock creation failed, just skip over index creation entirely.
throw new StingException("Index file does not exist and could not be created. See error below.", e);
}
finally {
@ -96,15 +103,13 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
*/
// get read lock on dict file so nobody else can read it
FSLockWithShared dictLock = new FSLockWithShared(dictFile);
FSLockWithShared dictLock = new FSLockWithShared(dictFile,true);
try {
// get shared lock on dict file so nobody else can start creating it
if (!dictLock.exclusiveLock())
throw new StingException("Dictionary file could not be written because a lock could not be obtained." +
"If you are running multiple instances of GATK, another process is probably creating this " +
"file now. Please wait until it is finished and try again.");
// dict will be written to random temporary file in same directory (see note above)
File tempFile = File.createTempFile("dict", null, dictFile.getParentFile());
tempFile.deleteOnExit();
@ -117,7 +122,13 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
if (!tempFile.renameTo(dictFile))
throw new StingException("Error transferring temp file to dict file");
}
catch(FileSystemInabilityToLockException ex) {
logger.info("Unable to create write lock: " + ex.getMessage());
logger.info("Skipping dictionary creation.");
}
catch (Exception e) {
// If lock creation succeeded, the failure must have been generating the index.
// If lock creation failed, just skip over index creation entirely.
throw new StingException("Dictionary file does not exist and could not be created. See error below.", e);
}
finally {
@ -132,14 +143,27 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
* but is incomplete). To avoid this, obtain shared locks on both files before creating IndexedFastaSequenceFile.
*/
FSLockWithShared dictLock = new FSLockWithShared(dictFile);
FSLockWithShared indexLock = new FSLockWithShared(indexFile);
FSLockWithShared dictLock = new FSLockWithShared(dictFile,true);
FSLockWithShared indexLock = new FSLockWithShared(indexFile,true);
try {
if (!dictLock.sharedLock()) {
throw new StingException("Could not open dictionary file because a lock could not be obtained.");
try {
if (!dictLock.sharedLock()) {
throw new StingException("Could not open dictionary file because a lock could not be obtained.");
}
}
if (!indexLock.sharedLock()) {
throw new StingException("Could not open dictionary file because a lock could not be obtained.");
catch(FileSystemInabilityToLockException ex) {
logger.info(String.format("Unable to create a lock on dictionary file: %s",ex.getMessage()));
logger.info("Treating existing dictionary file as complete.");
}
try {
if (!indexLock.sharedLock()) {
throw new StingException("Could not open index file because a lock could not be obtained.");
}
}
catch(FileSystemInabilityToLockException ex) {
logger.info(String.format("Unable to create a lock on index file: %s",ex.getMessage()));
logger.info("Treating existing index file as complete.");
}
index = new IndexedFastaSequenceFile(fastaFile);

15
java/src/org/broadinstitute/sting/gatk/refdata/tracks/builders/TribbleRMDTrackBuilder.java
View File

@ -45,6 +45,7 @@ import org.broadinstitute.sting.utils.collections.Pair;
import org.broadinstitute.sting.utils.classloader.PluginManager;
import org.broadinstitute.sting.utils.StingException;
import org.broadinstitute.sting.utils.file.FSLockWithShared;
import org.broadinstitute.sting.utils.file.FileSystemInabilityToLockException;
import java.io.*;
import java.util.*;
@ -208,7 +209,13 @@ public class TribbleRMDTrackBuilder extends PluginManager<FeatureCodec> implemen
* @throws IOException if we fail for FS issues
*/
protected static Index attemptIndexFromDisk(File inputFile, FeatureCodec codec, File indexFile, FSLockWithShared lock) throws IOException {
boolean locked = lock.sharedLock();
boolean locked;
try {
locked = lock.sharedLock();
}
catch(FileSystemInabilityToLockException ex) {
throw new StingException("Unexpected inability to lock exception", ex);
}
Index idx;
try {
if (!locked) // can't lock file
@ -272,7 +279,11 @@ public class TribbleRMDTrackBuilder extends PluginManager<FeatureCodec> implemen
else // we can't write it to disk, just store it in memory, tell them this
if (onDisk) logger.info("Unable to write to " + indexFile + " for the index file, creating index in memory only");
return index;
} finally {
}
catch(FileSystemInabilityToLockException ex) {
throw new StingException("Unexpected inability to lock exception", ex);
}
finally {
if (locked) lock.unlock();
}

31
java/src/org/broadinstitute/sting/utils/file/FSLockWithShared.java
View File

@ -26,6 +26,14 @@ public class FSLockWithShared {
// the file channel we open
private FileChannel channel = null;
/**
* A bit of experimental code for Siva at Partners. Conditionally throw an
* exception in the case where an unknown failure occurs, in an effort to stave
* off disabled nfs file locks.
*/
private boolean throwExceptionOnUnknownFailure = false;
/**
* create a file system, given a base file to which a lock string gets appended.
* @param baseFile File descriptor of file to lock
@ -34,12 +42,17 @@ public class FSLockWithShared {
file = baseFile;
}
public FSLockWithShared(File baseFile,boolean throwExceptionOnUnknownFailure) {
this(baseFile);
this.throwExceptionOnUnknownFailure = throwExceptionOnUnknownFailure;
}
/**
* Get a shared (read) lock on a file
* Cannot get shared lock if it does not exist
* @return boolean true if we obtained a lock
*/
public boolean sharedLock() {
public boolean sharedLock() throws FileSystemInabilityToLockException {
// get read-only file channel
try {
@ -66,8 +79,11 @@ public class FSLockWithShared {
return false;
}
catch (IOException e) {
logger.debug("Unable to lock file (due to IO exception)");
return false;
logger.debug("Unable to lock file: " + e.getMessage());
if(throwExceptionOnUnknownFailure)
throw new FileSystemInabilityToLockException(e.getMessage(),e);
else
return false;
}
return true;
}
@ -76,7 +92,7 @@ public class FSLockWithShared {
* Get an exclusive lock on a file
* @return boolean true if we obtained a lock
*/
public boolean exclusiveLock() {
public boolean exclusiveLock() throws FileSystemInabilityToLockException {
// read/write file channel is necessary for exclusive lock
try {
@ -106,8 +122,11 @@ public class FSLockWithShared {
return false;
}
catch (IOException e) {
logger.debug("Unable to lock file (due to IO exception)");
return false;
logger.debug("Unable to lock file: " + e.getMessage());
if(throwExceptionOnUnknownFailure)
throw new FileSystemInabilityToLockException(e.getMessage(),e);
else
return false;
}
}

46
java/src/org/broadinstitute/sting/utils/file/FileSystemInabilityToLockException.java 100644
View File

@ -0,0 +1,46 @@
/*
* Copyright (c) 2010, The Broad Institute
*
* Permission is hereby granted, free of charge, to any person
* obtaining a copy of this software and associated documentation
* files (the "Software"), to deal in the Software without
* restriction, including without limitation the rights to use,
* copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following
* conditions:
*
* The above copyright notice and this permission notice shall be
* included in all copies or substantial portions of the Software.
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
* OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
* HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
* WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*/
package org.broadinstitute.sting.utils.file;
/**
* A special checked exception that happens only in the case where
* the filesystem, by design or configuration, is completely unable
* to handle locking. This exception will specifically NOT be thrown
* in the case where the filesystem handles locking but is unable to
* acquire a lock due to concurrency.
*
* @author hanna
* @version 0.1
*/
public class FileSystemInabilityToLockException extends Exception {
/**
* Force user to create this exception with a nested inner stack trace.
* @param message Exception message.
* @param innerException Caused-by exception.
*/
public FileSystemInabilityToLockException(String message,Exception innerException) {
super(message,innerException);
}
}