Adding support for shared file locking via a new class for file locking, FSLockWithShared. This will eventually take over for FSLock, the current file locking class - I'll work with Aaron to merge the tribble code that uses FSLock right now.
FYI: creating an exclusive lock on a file that does not exist will create that file as an empty file, and will NOT delete that file after the program terminates. So watch out if it's possible that the file you're locking does not exist - could end up leaving extra files that confuse users. git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@3795 348d0f76-0448-11de-a6fe-93d51630548a
This commit is contained in:
bthomas
parent
a8caa20378
commit
f65cba6b9a
|
|
@ -30,11 +30,9 @@ import net.sf.picard.reference.FastaSequenceIndexBuilder;
|
|||
import net.sf.picard.sam.CreateSequenceDictionary;
|
||||
import net.sf.picard.reference.IndexedFastaSequenceFile;
|
||||
import net.sf.picard.reference.FastaSequenceIndex;
|
||||
import org.broadinstitute.sting.utils.file.FSLockWithShared;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileLock;
|
||||
|
||||
/**
|
||||
* Loads reference data from fasta file
|
||||
|
|
@ -52,39 +50,35 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
|
|||
*/
|
||||
public ReferenceDataSource(File fastaFile) {
|
||||
File indexFile = new File(fastaFile.getAbsolutePath() + ".fai");
|
||||
File dictFile = new File(fastaFile.getAbsolutePath().replace(".fasta", ".dict"));
|
||||
File dictFile;
|
||||
if (fastaFile.getAbsolutePath().endsWith("fa")) {
|
||||
dictFile = new File(fastaFile.getAbsolutePath().replace(".fa", ".dict"));
|
||||
}
|
||||
else
|
||||
dictFile = new File(fastaFile.getAbsolutePath().replace(".fasta", ".dict"));
|
||||
|
||||
/*
|
||||
if index file does not exist, create it manually
|
||||
*/
|
||||
if (!indexFile.exists()) {
|
||||
logger.info(String.format("Index file %s does not exist. Trying to create it now.", indexFile.getAbsolutePath()));
|
||||
FileChannel indexChannel;
|
||||
FileLock indexLock;
|
||||
FSLockWithShared indexLock = new FSLockWithShared(indexFile);
|
||||
try {
|
||||
// get exclusive lock
|
||||
indexChannel = new RandomAccessFile(indexFile, "rw").getChannel();
|
||||
if ((indexLock = indexChannel.tryLock(0, Long.MAX_VALUE, false)) == null)
|
||||
if (!indexLock.exclusiveLock())
|
||||
throw new StingException("Index file could not be written because a lock could not be obtained." +
|
||||
"If you are running multiple instances of GATK, another process is probably creating this " +
|
||||
"file now. Please wait until it is finished and try again.");
|
||||
FastaSequenceIndexBuilder faiBuilder = new FastaSequenceIndexBuilder(fastaFile, this);
|
||||
FastaSequenceIndex sequenceIndex = faiBuilder.createIndex();
|
||||
FastaSequenceIndexBuilder.saveAsFaiFile(sequenceIndex, indexFile);
|
||||
// unlock
|
||||
try {
|
||||
if (indexLock != null)
|
||||
indexLock.release();
|
||||
if (indexChannel != null)
|
||||
indexChannel.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("Index file does not exist and could not be created. See error below.", e);
|
||||
}
|
||||
finally {
|
||||
indexLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -94,36 +88,41 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
|
|||
* This has been filed in trac as (PIC-370) Want programmatic interface to CreateSequenceDictionary
|
||||
*/
|
||||
if (!dictFile.exists()) {
|
||||
logger.info(String.format("Index file %s does not exist. Trying to create it now.", indexFile.getAbsolutePath()));
|
||||
FileChannel dictChannel;
|
||||
FileLock dictLock;
|
||||
try {
|
||||
// get exclusive lock
|
||||
dictChannel = new RandomAccessFile(indexFile, "rw").getChannel();
|
||||
if ((dictLock = dictChannel.tryLock(0, Long.MAX_VALUE, false)) == null)
|
||||
logger.info(String.format("Dict file %s does not exist. Trying to create it now.", dictFile.getAbsolutePath()));
|
||||
|
||||
/*
|
||||
* Please note another hack here: we have to create a temporary file b/c CreateSequenceDictionary cannot
|
||||
* create a dictionary file if that file is locked.
|
||||
*/
|
||||
|
||||
// get read lock on dict file so nobody else can read it
|
||||
FSLockWithShared dictLock = new FSLockWithShared(dictFile);
|
||||
|
||||
try {
|
||||
// get shared lock on dict file so nobody else can start creating it
|
||||
if (!dictLock.exclusiveLock())
|
||||
throw new StingException("Dictionary file could not be written because a lock could not be obtained." +
|
||||
"If you are running multiple instances of GATK, another process is probably creating this " +
|
||||
"file now. Please wait until it is finished and try again.");
|
||||
|
||||
// dict will be written to random temporary file in same directory (see note above)
|
||||
File tempFile = File.createTempFile("dict", null, dictFile.getParentFile());
|
||||
tempFile.deleteOnExit();
|
||||
|
||||
// create dictionary by calling main routine. Temporary fix - see comment above.
|
||||
String args[] = {String.format("r=%s", fastaFile.getAbsolutePath()),
|
||||
String.format("o=%s", dictFile.getAbsolutePath())};
|
||||
String.format("o=%s", tempFile.getAbsolutePath())};
|
||||
new CreateSequenceDictionary().instanceMain(args);
|
||||
// unlock
|
||||
try {
|
||||
if (dictLock != null)
|
||||
dictLock.release();
|
||||
if (dictChannel != null)
|
||||
dictChannel.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e);
|
||||
}
|
||||
|
||||
if (!tempFile.renameTo(dictFile))
|
||||
throw new StingException("Error transferring temp file to dict file");
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("Dictionary file does not exist and could not be created. See error below.", e);
|
||||
}
|
||||
finally {
|
||||
dictLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -133,46 +132,25 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener
|
|||
* but is incomplete). To avoid this, obtain shared locks on both files before creating IndexedFastaSequenceFile.
|
||||
*/
|
||||
|
||||
FileChannel dictChannel;
|
||||
FileChannel indexChannel;
|
||||
FileLock dictLock;
|
||||
FileLock indexLock;
|
||||
FSLockWithShared dictLock = new FSLockWithShared(dictFile);
|
||||
FSLockWithShared indexLock = new FSLockWithShared(indexFile);
|
||||
try {
|
||||
// set up dictionary and index locks
|
||||
// channel is read only and lock is shared (third argument is true)
|
||||
dictChannel = new RandomAccessFile(dictFile, "r").getChannel();
|
||||
if ((dictLock = dictChannel.tryLock(0, Long.MAX_VALUE, true)) == null) {
|
||||
if (!dictLock.sharedLock()) {
|
||||
throw new StingException("Could not open dictionary file because a lock could not be obtained.");
|
||||
}
|
||||
indexChannel = new RandomAccessFile(indexFile, "r").getChannel();
|
||||
if ((indexLock = indexChannel.tryLock(0, Long.MAX_VALUE, true)) == null) {
|
||||
if (!indexLock.sharedLock()) {
|
||||
throw new StingException("Could not open dictionary file because a lock could not be obtained.");
|
||||
}
|
||||
|
||||
index = new IndexedFastaSequenceFile(fastaFile);
|
||||
|
||||
// unlock/close
|
||||
try {
|
||||
if (dictLock != null)
|
||||
dictLock.release();
|
||||
if (dictChannel != null)
|
||||
dictChannel.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("An error occurred while unlocking file:" + dictFile.getAbsolutePath(), e);
|
||||
}
|
||||
try {
|
||||
if (indexLock != null)
|
||||
indexLock.release();
|
||||
if (indexChannel != null)
|
||||
indexChannel.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e);
|
||||
}
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException(String.format("Error reading fasta file %s. See stack trace below.", fastaFile.getAbsolutePath()), e);
|
||||
throw new StingException(String.format("Error reading fasta file %s.", fastaFile.getAbsolutePath()), e);
|
||||
}
|
||||
finally {
|
||||
dictLock.unlock();
|
||||
indexLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
package org.broadinstitute.sting.utils.file;
|
||||
|
||||
import org.apache.log4j.Logger;
|
||||
import org.broadinstitute.sting.utils.StingException;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.RandomAccessFile;
|
||||
import java.nio.channels.ClosedChannelException;
|
||||
import java.nio.channels.FileChannel;
|
||||
import java.nio.channels.FileLock;
|
||||
import java.nio.channels.OverlappingFileLockException;
|
||||
|
||||
/**
|
||||
* a quick implementation of a file based lock, using the Java NIO classes
|
||||
*/
|
||||
public class FSLockWithShared {
|
||||
// connect to the logger
|
||||
private final static Logger logger = Logger.getLogger(FSLockWithShared.class);
|
||||
|
||||
// the file we're attempting to lock
|
||||
private final File file;
|
||||
|
||||
// the file lock
|
||||
private FileLock lock = null;
|
||||
|
||||
// the file channel we open
|
||||
private FileChannel channel = null;
|
||||
/**
|
||||
* create a file system, given a base file to which a lock string gets appended.
|
||||
* @param baseFile File descriptor of file to lock
|
||||
*/
|
||||
public FSLockWithShared(File baseFile) {
|
||||
file = baseFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a shared (read) lock on a file
|
||||
* Cannot get shared lock if it does not exist
|
||||
* @return boolean true if we obtained a lock
|
||||
*/
|
||||
public boolean sharedLock() {
|
||||
|
||||
// get read-only file channel
|
||||
try {
|
||||
channel = new RandomAccessFile(file, "r").getChannel();
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.debug("Unable to lock file (could not open read only file channel)");
|
||||
return false;
|
||||
}
|
||||
// get shared lock (third argument is true)
|
||||
try {
|
||||
lock = channel.tryLock(0, Long.MAX_VALUE, true);
|
||||
if (lock == null) {
|
||||
logger.debug("Unable to lock file because there is already a lock active.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (ClosedChannelException e) {
|
||||
logger.debug("Unable to lock file because the file channel is closed.");
|
||||
return false;
|
||||
}
|
||||
catch (OverlappingFileLockException e) {
|
||||
logger.debug("Unable to lock file because you already have a lock on this file.");
|
||||
return false;
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.debug("Unable to lock file (due to IO exception)");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an exclusive lock on a file
|
||||
* @return boolean true if we obtained a lock
|
||||
*/
|
||||
public boolean exclusiveLock() {
|
||||
|
||||
// read/write file channel is necessary for exclusive lock
|
||||
try {
|
||||
channel = new RandomAccessFile(file, "rw").getChannel();
|
||||
}
|
||||
catch (Exception e) {
|
||||
logger.debug("Unable to lock file (could not open read/write file channel)");
|
||||
// do we need to worry about deleting file here? Does RandomAccessFile will only create file if successful?
|
||||
return false;
|
||||
}
|
||||
|
||||
// get exclusive lock (third argument is false)
|
||||
try {
|
||||
lock = channel.tryLock(0, Long.MAX_VALUE, false);
|
||||
if (lock == null) {
|
||||
logger.debug("Unable to lock file because there is already a lock active.");
|
||||
return false;
|
||||
}
|
||||
else return true;
|
||||
}
|
||||
catch (ClosedChannelException e) {
|
||||
logger.debug("Unable to lock file because the file channel is closed.");
|
||||
return false;
|
||||
}
|
||||
catch (OverlappingFileLockException e) {
|
||||
logger.debug("Unable to lock file because you already have a lock on this file.");
|
||||
return false;
|
||||
}
|
||||
catch (IOException e) {
|
||||
logger.debug("Unable to lock file (due to IO exception)");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* unlock the file
|
||||
*
|
||||
* note: this allows unlocking a file that failed to lock (no required user checks on null locks).
|
||||
*/
|
||||
public void unlock() {
|
||||
try {
|
||||
if (lock != null)
|
||||
lock.release();
|
||||
if (channel != null)
|
||||
channel.close();
|
||||
}
|
||||
catch (Exception e) {
|
||||
throw new StingException("An error occurred while unlocking file", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue