diff --git a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceDataSource.java b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceDataSource.java index 4defcdd80..818ce5df9 100644 --- a/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceDataSource.java +++ b/java/src/org/broadinstitute/sting/gatk/datasources/simpleDataSources/ReferenceDataSource.java @@ -30,11 +30,9 @@ import net.sf.picard.reference.FastaSequenceIndexBuilder; import net.sf.picard.sam.CreateSequenceDictionary; import net.sf.picard.reference.IndexedFastaSequenceFile; import net.sf.picard.reference.FastaSequenceIndex; +import org.broadinstitute.sting.utils.file.FSLockWithShared; import java.io.File; -import java.io.RandomAccessFile; -import java.nio.channels.FileChannel; -import java.nio.channels.FileLock; /** * Loads reference data from fasta file @@ -52,39 +50,35 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener */ public ReferenceDataSource(File fastaFile) { File indexFile = new File(fastaFile.getAbsolutePath() + ".fai"); - File dictFile = new File(fastaFile.getAbsolutePath().replace(".fasta", ".dict")); + File dictFile; + if (fastaFile.getAbsolutePath().endsWith("fa")) { + dictFile = new File(fastaFile.getAbsolutePath().replace(".fa", ".dict")); + } + else + dictFile = new File(fastaFile.getAbsolutePath().replace(".fasta", ".dict")); /* if index file does not exist, create it manually */ if (!indexFile.exists()) { logger.info(String.format("Index file %s does not exist. Trying to create it now.", indexFile.getAbsolutePath())); - FileChannel indexChannel; - FileLock indexLock; + FSLockWithShared indexLock = new FSLockWithShared(indexFile); try { // get exclusive lock - indexChannel = new RandomAccessFile(indexFile, "rw").getChannel(); - if ((indexLock = indexChannel.tryLock(0, Long.MAX_VALUE, false)) == null) + if (!indexLock.exclusiveLock()) throw new StingException("Index file could not be written because a lock could not be obtained." + "If you are running multiple instances of GATK, another process is probably creating this " + "file now. Please wait until it is finished and try again."); FastaSequenceIndexBuilder faiBuilder = new FastaSequenceIndexBuilder(fastaFile, this); FastaSequenceIndex sequenceIndex = faiBuilder.createIndex(); FastaSequenceIndexBuilder.saveAsFaiFile(sequenceIndex, indexFile); - // unlock - try { - if (indexLock != null) - indexLock.release(); - if (indexChannel != null) - indexChannel.close(); - } - catch (Exception e) { - throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e); - } } catch (Exception e) { throw new StingException("Index file does not exist and could not be created. See error below.", e); } + finally { + indexLock.unlock(); + } } /* @@ -94,36 +88,41 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener * This has been filed in trac as (PIC-370) Want programmatic interface to CreateSequenceDictionary */ if (!dictFile.exists()) { - logger.info(String.format("Index file %s does not exist. Trying to create it now.", indexFile.getAbsolutePath())); - FileChannel dictChannel; - FileLock dictLock; - try { - // get exclusive lock - dictChannel = new RandomAccessFile(indexFile, "rw").getChannel(); - if ((dictLock = dictChannel.tryLock(0, Long.MAX_VALUE, false)) == null) + logger.info(String.format("Dict file %s does not exist. Trying to create it now.", dictFile.getAbsolutePath())); + /* + * Please note another hack here: we have to create a temporary file b/c CreateSequenceDictionary cannot + * create a dictionary file if that file is locked. + */ + + // get read lock on dict file so nobody else can read it + FSLockWithShared dictLock = new FSLockWithShared(dictFile); + + try { + // get shared lock on dict file so nobody else can start creating it + if (!dictLock.exclusiveLock()) throw new StingException("Dictionary file could not be written because a lock could not be obtained." + "If you are running multiple instances of GATK, another process is probably creating this " + "file now. Please wait until it is finished and try again."); + // dict will be written to random temporary file in same directory (see note above) + File tempFile = File.createTempFile("dict", null, dictFile.getParentFile()); + tempFile.deleteOnExit(); + // create dictionary by calling main routine. Temporary fix - see comment above. String args[] = {String.format("r=%s", fastaFile.getAbsolutePath()), - String.format("o=%s", dictFile.getAbsolutePath())}; + String.format("o=%s", tempFile.getAbsolutePath())}; new CreateSequenceDictionary().instanceMain(args); - // unlock - try { - if (dictLock != null) - dictLock.release(); - if (dictChannel != null) - dictChannel.close(); - } - catch (Exception e) { - throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e); - } + + if (!tempFile.renameTo(dictFile)) + throw new StingException("Error transferring temp file to dict file"); } catch (Exception e) { throw new StingException("Dictionary file does not exist and could not be created. See error below.", e); } + finally { + dictLock.unlock(); + } } /* @@ -133,46 +132,25 @@ public class ReferenceDataSource implements ReferenceDataSourceProgressListener * but is incomplete). To avoid this, obtain shared locks on both files before creating IndexedFastaSequenceFile. */ - FileChannel dictChannel; - FileChannel indexChannel; - FileLock dictLock; - FileLock indexLock; + FSLockWithShared dictLock = new FSLockWithShared(dictFile); + FSLockWithShared indexLock = new FSLockWithShared(indexFile); try { - // set up dictionary and index locks - // channel is read only and lock is shared (third argument is true) - dictChannel = new RandomAccessFile(dictFile, "r").getChannel(); - if ((dictLock = dictChannel.tryLock(0, Long.MAX_VALUE, true)) == null) { + if (!dictLock.sharedLock()) { throw new StingException("Could not open dictionary file because a lock could not be obtained."); } - indexChannel = new RandomAccessFile(indexFile, "r").getChannel(); - if ((indexLock = indexChannel.tryLock(0, Long.MAX_VALUE, true)) == null) { + if (!indexLock.sharedLock()) { throw new StingException("Could not open dictionary file because a lock could not be obtained."); } index = new IndexedFastaSequenceFile(fastaFile); - // unlock/close - try { - if (dictLock != null) - dictLock.release(); - if (dictChannel != null) - dictChannel.close(); - } - catch (Exception e) { - throw new StingException("An error occurred while unlocking file:" + dictFile.getAbsolutePath(), e); - } - try { - if (indexLock != null) - indexLock.release(); - if (indexChannel != null) - indexChannel.close(); - } - catch (Exception e) { - throw new StingException("An error occurred while unlocking file:" + indexFile.getAbsolutePath(), e); - } } catch (Exception e) { - throw new StingException(String.format("Error reading fasta file %s. See stack trace below.", fastaFile.getAbsolutePath()), e); + throw new StingException(String.format("Error reading fasta file %s.", fastaFile.getAbsolutePath()), e); + } + finally { + dictLock.unlock(); + indexLock.unlock(); } } diff --git a/java/src/org/broadinstitute/sting/utils/file/FSLockWithShared.java b/java/src/org/broadinstitute/sting/utils/file/FSLockWithShared.java new file mode 100644 index 000000000..4fd8d35e5 --- /dev/null +++ b/java/src/org/broadinstitute/sting/utils/file/FSLockWithShared.java @@ -0,0 +1,130 @@ +package org.broadinstitute.sting.utils.file; + +import org.apache.log4j.Logger; +import org.broadinstitute.sting.utils.StingException; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; + +/** + * a quick implementation of a file based lock, using the Java NIO classes + */ +public class FSLockWithShared { + // connect to the logger + private final static Logger logger = Logger.getLogger(FSLockWithShared.class); + + // the file we're attempting to lock + private final File file; + + // the file lock + private FileLock lock = null; + + // the file channel we open + private FileChannel channel = null; + /** + * create a file system, given a base file to which a lock string gets appended. + * @param baseFile File descriptor of file to lock + */ + public FSLockWithShared(File baseFile) { + file = baseFile; + } + + /** + * Get a shared (read) lock on a file + * Cannot get shared lock if it does not exist + * @return boolean true if we obtained a lock + */ + public boolean sharedLock() { + + // get read-only file channel + try { + channel = new RandomAccessFile(file, "r").getChannel(); + } + catch (IOException e) { + logger.debug("Unable to lock file (could not open read only file channel)"); + return false; + } + // get shared lock (third argument is true) + try { + lock = channel.tryLock(0, Long.MAX_VALUE, true); + if (lock == null) { + logger.debug("Unable to lock file because there is already a lock active."); + return false; + } + } + catch (ClosedChannelException e) { + logger.debug("Unable to lock file because the file channel is closed."); + return false; + } + catch (OverlappingFileLockException e) { + logger.debug("Unable to lock file because you already have a lock on this file."); + return false; + } + catch (IOException e) { + logger.debug("Unable to lock file (due to IO exception)"); + return false; + } + return true; + } + + /** + * Get an exclusive lock on a file + * @return boolean true if we obtained a lock + */ + public boolean exclusiveLock() { + + // read/write file channel is necessary for exclusive lock + try { + channel = new RandomAccessFile(file, "rw").getChannel(); + } + catch (Exception e) { + logger.debug("Unable to lock file (could not open read/write file channel)"); + // do we need to worry about deleting file here? Does RandomAccessFile will only create file if successful? + return false; + } + + // get exclusive lock (third argument is false) + try { + lock = channel.tryLock(0, Long.MAX_VALUE, false); + if (lock == null) { + logger.debug("Unable to lock file because there is already a lock active."); + return false; + } + else return true; + } + catch (ClosedChannelException e) { + logger.debug("Unable to lock file because the file channel is closed."); + return false; + } + catch (OverlappingFileLockException e) { + logger.debug("Unable to lock file because you already have a lock on this file."); + return false; + } + catch (IOException e) { + logger.debug("Unable to lock file (due to IO exception)"); + return false; + } + } + + /** + * unlock the file + * + * note: this allows unlocking a file that failed to lock (no required user checks on null locks). + */ + public void unlock() { + try { + if (lock != null) + lock.release(); + if (channel != null) + channel.close(); + } + catch (Exception e) { + throw new StingException("An error occurred while unlocking file", e); + } + } +}