inputFiles) {
- mInputFiles = inputFiles;
- }
-
- public void close() {
- if (mCurrentReader != null) {
- try {
- mCurrentReader.close();
- } catch (IOException exc) {
- throw new RuntimeException("Error closing reader: " + exc.getMessage(),
- exc);
- }
- }
- mCurrentReader = null;
- mInputFiles = null;
- mInputFileIndex = 0;
- mBaseIndex = -1;
- mNextSequence = null;
- mLineBuffer = null;
- mLineBufferIndex = 0;
- }
-
- public String getNextSequence()
- throws IOException {
-
- while (mNextSequence == null) {
- if (mLineBuffer != null) {
- incrementBaseIndex(mLineBuffer.length() - mLineBufferIndex);
- mLineBuffer = null;
- mLineBufferIndex = 0;
- }
- if (mCurrentReader == null) {
- mCurrentReader = getNextReader();
- if (mCurrentReader == null) {
- return null;
- }
- }
- String line = mCurrentReader.readLine();
- if (line == null) {
- mCurrentReader.close();
- mCurrentReader = null;
- continue;
- }
- if (line.startsWith(">")) {
- String[] tokens = line.substring(1).trim().split("\\s+");
- mNextSequence = tokens[0];
- } else {
- incrementBaseIndex(line.length());
- }
- }
- String result = mNextSequence;
- mNextSequence = null;
- return result;
- }
-
- public char getNextBase()
- throws IOException {
-
- if (mLineBuffer == null || mLineBufferIndex >= mLineBuffer.length()) {
- if (mCurrentReader == null) {
- return 0;
- }
- if (mNextSequence != null) {
- return 0;
- }
- String line = mCurrentReader.readLine();
- if (line == null) {
- mLineBuffer = null;
- mLineBufferIndex = 0;
- mCurrentReader.close();
- mCurrentReader = null;
- return 0;
- }
- if (line.startsWith(">")) {
- String[] tokens = line.substring(1).trim().split("\\s+");
- mNextSequence = tokens[0];
- mLineBuffer = null;
- mLineBufferIndex = 0;
- return 0;
- }
- mLineBuffer = line.toUpperCase();
- mLineBufferIndex = 0;
- }
- char result = mLineBuffer.charAt(mLineBufferIndex++);
- incrementBaseIndex(1);
- return result;
- }
-
- public int getBaseIndex() {
- return mBaseIndex;
- }
-
- private LineNumberReader getNextReader()
- throws IOException {
- if (mInputFileIndex >= mInputFiles.size()) {
- return null;
- }
- File file = mInputFiles.get(mInputFileIndex++);
- return new LineNumberReader(new FileReader(file));
- }
-
- private void incrementBaseIndex(int amount) {
- if (mBaseIndex < -1 && (mBaseIndex + amount) >= -1) {
- throw new RuntimeException("Base index: 32-bit overflow");
- }
- mBaseIndex += amount;
- }
-}
-
diff --git a/java/lib/edu/mit/broad/dcp/CallStatus.java b/java/lib/edu/mit/broad/dcp/CallStatus.java
deleted file mode 100644
index e431b27df..000000000
--- a/java/lib/edu/mit/broad/dcp/CallStatus.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2007 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-public enum CallStatus
-{
- PENDING,
- PROCESSING
-}
-
-
diff --git a/java/lib/edu/mit/broad/dcp/CommandRunner.java b/java/lib/edu/mit/broad/dcp/CommandRunner.java
deleted file mode 100644
index b93b310dd..000000000
--- a/java/lib/edu/mit/broad/dcp/CommandRunner.java
+++ /dev/null
@@ -1,309 +0,0 @@
-/**
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2006 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-import java.io.*;
-
-
-/**
- * Utility class to run system commands synchronously and return the output.
- *
- * The interface supports the typical case where you want to return a modest
- * amount of information from the command's standard output or standard error
- * as a string. The caller can override this behavior, however, and provide
- * alternative output destinations if necessary.
- *
- * If setMergeOutput() is true, then this class will attempt to interleave
- * the standard output and standard error streams of the command into one
- * stream (standard output). This may not produce exactly the same results
- * as having the operating system interleave the output, but works well for
- * simple executables that do not heavily intermix stdout and stderr.
- *
- * A typical invocation is:
- *
- * CommandRunner runner = new CommandRunner();
- * int status = runner.runCommand("ls");
- * if (status == 0) {
- * System.out.print(runner.getStandardOutput());
- * }
- *
- *
- * @author Bob Handsaker
- */
-public class CommandRunner {
-
- private boolean mMergeOutput = false;
- private Writer mStandardOutputDestination = null;
- private Writer mStandardErrorDestination = null;
- private String mStandardOutputString = null;
- private String mStandardErrorString = null;
-
-
- /**
- * Default constructor.
- */
- public CommandRunner() {
- }
-
- /**
- * Get the standard output from the last command as a string.
- *
- * If no command has been run or an explicit output destination
- * was set, then this method returns null.
- */
- public String getStandardOutputString() {
- return mStandardOutputString;
- }
-
- /**
- * Get the standard error from the last command as a string.
- *
- * If no command has been run or an explicit output destination
- * was set, then this method returns null.
- */
- public String getStandardErrorString() {
- return mStandardErrorString;
- }
-
- /**
- * If true, the command's standard error stream will be interleaved
- * with the command's standard output stream. The standard error
- * stream destination will not be used.
- */
- public boolean getMergeOutput() {
- return mMergeOutput;
- }
-
- /**
- * If true, the command's standard error stream will be interleaved
- * with the command's standard output stream.
- */
- public void setMergeOutput(boolean value) {
- mMergeOutput = value;
- }
-
- /**
- * The destination for the command's standard output stream.
- * If null, the standard output will be captured in a string.
- */
- public Writer getStandardOutputDestination() {
- return mStandardOutputDestination;
- }
-
- /**
- * The destination for the command's standard output stream.
- * If set to null, the standard output will be captured in a string.
- */
- public void setStandardOutputDestination(Writer writer) {
- mStandardOutputDestination = writer;
- }
-
- /**
- * The destination for the command's standard error stream.
- * If null, the standard error will be captured in a string.
- */
- public Writer getStandardErrorDestination() {
- return mStandardErrorDestination;
- }
-
- /**
- * The destination for the command's standard error stream.
- * If set to null, the standard error will be captured in a string.
- */
- public void setStandardErrorDestination(Writer writer) {
- mStandardErrorDestination = writer;
- }
-
- /**
- * Run a command string as a system command.
- *
- * Returns the exit status of the command.
- *
- * When this method is called, the standard output string
- * and standard error string are updated if no alternative output
- * destinations have been set.
- *
- * This method throws a RuntimeException if running the command fails
- * (for example, if there are not enough system resources to spawn
- * the process).
- *
- * @param commmand The command string to run.
- * @return Command exit status.
- * @throws RuntimeException If command execution fails.
- */
- public int runCommand(String command)
- throws RuntimeException {
- return runCommand(command.split(" "), null, null);
- }
-
- /**
- * Run a command string as a system command.
- *
- * Returns the exit status of the command.
- *
- * When this method is called, the standard output string
- * and standard error string are updated if no alternative output
- * destinations have been set.
- *
- * This method throws a RuntimeException if running the command fails
- * (for example, if there are not enough system resources to spawn
- * the process).
- *
- * @param commmand The command string to run.
- * @param environment The command environment (or null to inherit).
- * @param workingDirectory The working directory (or null to inherit).
- * @return Command exit status.
- * @throws RuntimeException If command execution fails.
- */
- public int runCommand(String command, String[] environment, File workingDirectory)
- throws RuntimeException {
- return runCommand(command.split(" "), environment, workingDirectory);
- }
-
- /**
- * Run a command string as a system command.
- *
- * Returns the exit status of the command.
- *
- * When this method is called, the standard output string
- * and standard error string are updated if no alternative output
- * destinations have been set.
- *
- * This method throws a RuntimeException if running the command fails
- * (for example, if there are not enough system resources to spawn
- * the process).
- *
- * @param commmand The command to run (as a array of arguments).
- * @param environment The command environment (or null to inherit).
- * @param workingDirectory The working directory (or null to inherit).
- * @return Command exit status.
- * @throws RuntimeException If command execution fails.
- */
- public int runCommand(String[] command, String[] environment, File workingDirectory)
- throws RuntimeException {
-
- Writer stdout = mStandardOutputDestination;
- Writer stderr = mStandardErrorDestination;
- if (stdout == null) {
- stdout = new StringWriter();
- }
- if (mMergeOutput) {
- stderr = stdout;
- } else if (stderr == null) {
- stderr = new StringWriter();
- }
-
- mStandardOutputString = null;
- mStandardErrorString = null;
-
- int commandStatus = 0;
- try {
- Process process =
- Runtime.getRuntime().exec(command, environment, workingDirectory);
- StreamHandler stdoutHandler =
- new StreamHandler(process.getInputStream(), stdout);
- StreamHandler stderrHandler =
- new StreamHandler(process.getErrorStream(), stderr);
-
- commandStatus = process.waitFor();
-
- // Wait for the streams to drain.
- stdoutHandler.join();
- stderrHandler.join();
- } catch (Exception exc) {
- throw new RuntimeException("Command execution failed: " +
- exc.getMessage(),
- exc);
- }
-
- if (mStandardOutputDestination == null) {
- mStandardOutputString = stdout.toString();
- }
- if (mStandardErrorDestination == null && !mMergeOutput) {
- mStandardErrorString = stderr.toString();
- }
-
- return commandStatus;
- }
-
-
- /**
- * Internal class to asynchronously read from the standard output
- * and standard error streams of the command being executed.
- *
- * If you do not handle command output asynchronously, then execution
- * of a command may block in some environments if the program produces
- * too much output. In this case, the call to run the process will
- * never complete.
- */
- private static class StreamHandler extends Thread {
-
- /**
- * Constructor.
- * Create an instance of this class, which is an asynchronous
- * thread that will consume input from the given input stream
- * and send the output to the given output destination.
- *
- * @param input The input stream to read.
- * @param output The output destination.
- */
- StreamHandler(InputStream input, Writer output) {
- m_input = input;
- m_output = output;
- start();
- }
-
-
- /**
- * Standard thread run method.
- * Pipe input from the input source to the output destination
- * until there is no more input left.
- *
- * If an IOException occurs, the thread will make sure all
- * available output has been flushed to the destination and
- * then terminate. The IOException is not propagated.
- */
- public void run() {
-
- char[] buffer = new char[4096];
- Reader reader =
- new InputStreamReader(new BufferedInputStream(m_input));
- Writer writer = m_output;
-
- try {
- while (true) {
- int count = reader.read(buffer);
- if (count <= 0) {
- break;
- }
- if (writer != null) {
- synchronized (writer) {
- writer.write(buffer, 0, count);
- }
- }
- }
- } catch (IOException ignore) {
- // Ignore IO exceptions
- } finally {
- try {
- reader.close();
- } catch (Exception ignore) {
- }
- try {
- m_output.flush();
- } catch (Exception ignore) {
- }
- }
- }
-
- private InputStream m_input;
- private Writer m_output;
- }
-}
diff --git a/java/lib/edu/mit/broad/dcp/DistributedAlgorithm.java b/java/lib/edu/mit/broad/dcp/DistributedAlgorithm.java
deleted file mode 100644
index a223c0326..000000000
--- a/java/lib/edu/mit/broad/dcp/DistributedAlgorithm.java
+++ /dev/null
@@ -1,618 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2007 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-import edu.mit.broad.dcp.message.*;
-
-import java.io.*;
-import java.util.*;
-import java.lang.reflect.Method;
-import java.net.InetAddress;
-import java.net.ServerSocket;
-import java.rmi.registry.*;
-
-/**
- * Experimental.
- */
-public abstract class DistributedAlgorithm
- implements Serializable
-{
- public static final Integer ANY = 0;
- public static final Integer MASTER = 1;
-
- public DistributedAlgorithm() {
- }
-
- public String getServerHost() {
- return mServerHost;
- }
-
- public void setServerHost(String value) {
- mServerHost = value;
- }
-
- public int getServerPort() {
- return mServerPort;
- }
-
- public void setServerPort(int value) {
- mServerPort = value;
- }
-
- public String getAlgorithmName() {
- if (mAlgorithmName != null) {
- return mAlgorithmName;
- } else {
- return getClassName();
- }
- }
-
- public void setAlgorithmName(String value) {
- mAlgorithmName = value;
- }
-
- public int getMaximumWorkerCount() {
- return mMaximumWorkerCount;
- }
-
- public void setMaximumWorkerCount(int value) {
- mMaximumWorkerCount = value;
- }
-
- /**
- * Name of LSF queue to use for workers.
- */
- public String getLsfQueue() {
- return mLsfQueue;
- }
-
- public void setLsfQueue(String value) {
- mLsfQueue = value;
- }
-
- /**
- * Directory to hold lsf log files.
- */
- public String getLsfLogDirectory() {
- return mLsfLogDirectory;
- }
-
- public void setLsfLogDirectory(String value) {
- mLsfLogDirectory = value;
- }
-
- public boolean getEnableGcLogging() {
- return mEnableGcLogging;
- }
-
- public void setEnableGcLogging(boolean value) {
- mEnableGcLogging = value;
- }
-
- public Integer getWorkerId() {
- return mWorkerId;
- }
-
- public Integer getProcessId() {
- return mProcessId;
- }
-
- protected void init()
- throws Exception {
- }
-
- protected abstract void start()
- throws Exception;
-
- public void run()
- throws Exception {
-
- if (mIsRunning) {
- throw new IllegalStateException("Algorithm is already running");
- }
-
- mIsRunning = true;
- mWorkerId = MASTER;
- mProcessId = MASTER;
-
- try {
- startDistributedServer();
- init();
- startWorkerThread();
- startWorkers();
- start();
- waitForCompletion();
- } finally {
- // TBD: More cleanup (shutdown threads, etc.)
- stopDistributedServer();
- mIsRunning = false;
- }
- }
-
- void runWorker(int workerId, int processId)
- throws Exception {
-
- if (mIsRunning) {
- throw new IllegalStateException("Algorithm is already running");
- }
-
- mIsRunning = true;
- mWorkerId = workerId;
- mProcessId = processId;
-
- try {
- if (openDistributedServer() == null) {
- report("Server " + mServerHost + ":" + mServerPort + " not responding");
- return;
- }
- init();
- startWorkerThread();
- mWorkerThread.join();
- } finally {
- closeDistributedServer();
- mIsRunning = false;
- }
- }
-
- private void startWorkers() {
- int workerCount = getMaximumWorkerCount();
- if (workerCount <= 0) {
- // Use single process execution for testing/debugging.
- new InProcessWorker().start();
- return;
- }
- if (workerCount > 1000) {
- throw new RuntimeException("Excessive worker count: " + workerCount);
- }
- for (int i = 0; i < workerCount; i++) {
- Integer workerId = (MASTER + i + 1);
- Integer processId = workerId; // for now
- startWorker(workerId, processId);
- }
- }
-
- private void startDistributedServer() {
- try {
- // Create a server socket to allocate a unique port.
- // There is a window of vulnerability where the port
- // can get reused, but in practice this works ok.
- String serverHost = getCurrentHost();
- ServerSocket socket = new ServerSocket(0);
- int serverPort = socket.getLocalPort();
- socket.close();
- Registry registry = LocateRegistry.createRegistry(serverPort);
- DistributedCallServer server = new DistributedCallServer();
- server.setAlgorithm(this);
- registry.bind("DistributedCallService", server);
- mServerHost = serverHost;
- mServerPort = serverPort;
- mDistributedCallServer = server;
- mDistributedCallService = server;
- } catch (Exception exc) {
- throw wrapException(exc);
- }
- }
-
- private void stopDistributedServer() {
- if (mDistributedCallServer != null) {
- try {
- Registry registry = LocateRegistry.getRegistry(mServerPort);
- registry.unbind("DistributedCallService");
- mDistributedCallServer.stop();
- } catch (Exception exc) {
- throw wrapException(exc);
- }
- }
- mDistributedCallService = null;
- mDistributedCallServer = null;
- }
-
- private DistributedCallService openDistributedServer() {
- mDistributedCallService = null;
- try {
- String url = "rmi://" + getServerHost() + ":" + getServerPort() + "/DistributedCallService";
- DistributedCallService server =
- (DistributedCallService) java.rmi.Naming.lookup(url);
- mDistributedCallService = server;
- } catch (java.rmi.NotBoundException exc) {
- // Server has exited
- } catch (Exception exc) {
- throw wrapException(exc);
- }
- return mDistributedCallService;
- }
-
- private void closeDistributedServer() {
- mDistributedCallService = null;
- }
-
- private void startWorker(Integer workerId, Integer processId) {
-
- String logFile = "worker_" + processId + "_%J.bsub";
- if (mLsfLogDirectory != null) {
- logFile = mLsfLogDirectory + "/" + logFile;
- }
-
- List command = new ArrayList();
- command.add("bsub");
- command.add("-o");
- command.add(logFile);
- if (mLsfQueue != null) {
- command.add("-q");
- command.add(mLsfQueue);
- }
- command.add("runDistributedWorker");
- command.add("-serverHost");
- command.add(getServerHost());
- command.add("-serverPort");
- command.add(Integer.toString(getServerPort()));
- command.add("-workerId");
- command.add(Integer.toString(workerId));
- command.add("-processId");
- command.add(Integer.toString(processId));
-
- // Pass our -Xmx setting along to all workers.
- Map environment =
- new LinkedHashMap(System.getenv());
- long maxMemory = Runtime.getRuntime().maxMemory();
- long maxKbytes = maxMemory / 1024;
- String memJavaOpt = "-Xmx" + maxKbytes + "K";
-
- // Enable GC logging if requested
- String gcJavaOpt = null;
- if (mEnableGcLogging) {
- String gcLogFile = "worker_" + processId + ".gc.log";
- if (mLsfLogDirectory != null) {
- gcLogFile = mLsfLogDirectory + "/" + gcLogFile;
- }
- gcJavaOpt = "-Xloggc:" + gcLogFile;
- }
-
- String javaOpts = environment.get("JAVAOPTS");
- if (javaOpts == null) {
- javaOpts = memJavaOpt;
- if (gcJavaOpt != null) {
- javaOpts = javaOpts + " " + gcJavaOpt;
- }
- environment.put("JAVAOPTS", javaOpts);
- }
-
- // Log output ourselves (rather than waiting for bsub).
- String workerLogFile = "worker_" + processId + ".log";
- if (mLsfLogDirectory != null) {
- workerLogFile = mLsfLogDirectory + "/" + workerLogFile;
- }
- environment.put("DA_LOG_FILE", workerLogFile);
-
- CommandRunner runner = new CommandRunner();
- Writer output = new LsfOutputFilter();
- runner.setStandardOutputDestination(output);
- runner.setStandardErrorDestination(output);
- String[] commandArray = command.toArray(new String[command.size()]);
- String[] environmentArray = createEnvironmentArray(environment);
- int status = runner.runCommand(commandArray, environmentArray, null);
- if (status != 0) {
- throw new RuntimeException("Error starting worker: " + status);
- }
- }
-
- private String[] createEnvironmentArray(Map map) {
- if (map == null) {
- return null;
- }
- int index = 0;
- String[] array = new String[map.size()];
- for (Map.Entry entry : map.entrySet()) {
- array[index++] = entry.getKey() + "=" + entry.getValue();
- }
- return array;
- }
-
- private String getCurrentHost() {
- try {
- return InetAddress.getLocalHost().getCanonicalHostName();
- } catch (Exception exc) {
- throw wrapException(exc);
- }
- }
-
- private void waitForCompletion() {
- DistributedCallServer server = mDistributedCallServer;
- while (true) {
- if (server.isQueueEmpty()) {
- break;
- }
- try {
- Thread.sleep(1000);
- } catch (InterruptedException exc) {
- // ignore
- }
- }
- }
-
- protected void callDistributed(String methodName, Object... methodArgs) {
- callDistributed(null, methodName, methodArgs);
- }
-
- protected void callDistributed(Integer workerId, String methodName, Object... methodArgs) {
- if (workerId == null) {
- workerId = ANY;
- }
- try {
- DistributedCallMessage message = new DistributedCallMessage();
- message.setSenderWorkerId(getWorkerId());
- message.setSenderProcessId(getProcessId());
- message.setReceiverWorkerId(workerId);
- message.setMethodName(methodName);
- message.setMethodArgs(methodArgs);
- mDistributedCallService.writeMessage(message);
- } catch (Throwable exc) {
- throw wrapException(exc);
- }
- }
-
- private void callMethod(String methodName, Object[] methodArgs) {
- try {
- Object target = this;
- Class targetClass = target.getClass();
- Method targetMethod = findMethod(targetClass, methodName);
- if (targetMethod == null) {
- throw new RuntimeException("Cannot find target method: " + methodName);
- }
- targetMethod.invoke(target, methodArgs);
- } catch (Throwable exc) {
- throw wrapException(exc);
- }
- }
-
- private Method findMethod(Class clazz, String methodName) throws Exception {
- Method result = null;
- Method[] methods = clazz.getDeclaredMethods();
- for (int i = 0; i < methods.length; i++) {
- if (methods[i].getName().equals(methodName)) {
- if (result != null) {
- throw new RuntimeException("Duplicate method name: " + methodName);
- }
- result = methods[i];
- }
- }
- return result;
- }
-
- private RuntimeException wrapException(Throwable exception) {
- if (exception instanceof RuntimeException) {
- return (RuntimeException) exception;
- } else {
- return new RuntimeException(exception.getMessage(), exception);
- }
- }
-
- private void startWorkerThread() {
- if (mWorkerThread != null) {
- throw new IllegalStateException("WorkerThread is running");
- }
- mWorkerThread = new WorkerThread();
- mWorkerThread.start();
- }
-
- private void stopWorkerThread() {
- if (mWorkerThread == null) {
- throw new IllegalStateException("WorkerThread is running");
- }
- mWorkerThread.stopThread();
- }
-
- private class WorkerThread extends Thread {
-
- WorkerThread() {
- setDaemon(true);
- }
-
- public void run() {
- try {
- DistributedCallService service = mDistributedCallService;
- while (true) {
- if (isInterrupted()) {
- System.out.println("#DBG: Worker isInterrupted");
- throw new InterruptedException();
- }
- DistributedCallMessage message =
- service.acceptMessage(getWorkerId(), getProcessId());
- if (message == null) {
- Thread.sleep(1000);
- } else {
- processMessage(message);
- }
- }
- } catch (InterruptedException exc) {
- // Interruption terminates this thread.
- // System.out.println("#DBG: Worker caught InterruptedException");
- } catch (Throwable exc) {
- if (isDisconnectException(exc)) {
- report("Server disconnected");
- } else {
- reportError("Exception in WorkerThread: " + exc.getMessage(), exc);
- System.exit(1);
- }
- }
- report("WorkerThread terminated");
- }
-
- void stopThread() {
- // System.out.println("#DBG: About to interrupt worker...");
- interrupt();
- // System.out.println("#DBG: Joining worker...");
- try {
- join();
- } catch (InterruptedException exc) {
- // ignore
- }
- }
-
- private boolean isDisconnectException(Throwable exc) {
- if (exc instanceof java.rmi.ConnectException) {
- return true;
- } else if (exc instanceof java.rmi.NoSuchObjectException) {
- return true;
- } else if (exc instanceof java.rmi.UnmarshalException &&
- exc.getCause() != null &&
- exc.getCause() instanceof EOFException) {
- return true;
- } else {
- return false;
- }
- }
- }
-
- private void processMessage(DistributedCallMessage message) {
- try {
- Integer workerId = message.getReceiverWorkerId();
- if (workerId == null || !workerId.equals(getWorkerId())) {
- reportError("Invalid worker ID in message: " + message);
- return;
- }
- callMethod(message.getMethodName(), message.getMethodArgs());
- } catch (Throwable exc) {
- reportError("Exception running message: " + message, exc);
- } finally {
- completeMessage(message);
- }
- }
-
- private void completeMessage(DistributedCallMessage message) {
- try {
- DistributedCallService service = mDistributedCallService;
- service.completeMessage(getWorkerId(), getProcessId(), message.getCallId());
- } catch (Throwable exc) {
- reportError("Exception completing message: " + message, exc);
- }
- }
-
- protected void report(String message) {
- String identity =
- getAlgorithmName() + " " +
- getWorkerId() + "/" + getProcessId();
- System.out.println("# " + identity + " : " + message);
- }
-
- protected void reportError(String message) {
- reportError(message, null);
- }
-
- protected void reportError(String message, Throwable exception) {
- String identity =
- getAlgorithmName() + " " +
- getWorkerId() + "/" + getProcessId();
- System.out.println("Error" +
- " [" + identity + "]" +
- ": " + message);
- if (exception != null) {
- System.out.println(" with exception: " + exception.getMessage());
- exception.printStackTrace(System.out);
- }
- }
-
- private String getClassName() {
- String name = getClass().getName();
- return name.substring(name.lastIndexOf('.')+1);
- }
-
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("DistributedAlgorithm");
- builder.append("(");
- builder.append("" + getAlgorithmName());
- builder.append(",");
- builder.append("" + getWorkerId());
- builder.append(",");
- builder.append("" + getProcessId());
- builder.append(",");
- builder.append("" + getMaximumWorkerCount());
- builder.append(",");
- builder.append("" + getLsfQueue());
- builder.append(",");
- builder.append("" + mIsRunning);
- builder.append(")");
- return builder.toString();
- }
-
- // This class is used only during in-process execution/testing/debugging.
- private class InProcessWorker extends Thread {
-
- InProcessWorker() {
- setDaemon(true);
- }
-
- public void run() {
- report("InProcessWorker starting");
- try {
- String serverAddress = getServerHost() + ":" + getServerPort();
- String url = "rmi://" + serverAddress + "/DistributedCallService";
- DistributedCallService server =
- (DistributedCallService) java.rmi.Naming.lookup(url);
- DistributedAlgorithm algorithm = server.getAlgorithm();
- algorithm.setServerHost(getServerHost());
- algorithm.setServerPort(getServerPort());
- algorithm.runWorker(2, 1);
- } catch (Throwable exc) {
- reportError("Exception in InProcessWorker: " + exc.getMessage(), exc);
- System.exit(1);
- }
- report("InProcessWorker terminated");
- }
- }
-
- private static class LsfOutputFilter
- extends FilterWriter {
-
- LsfOutputFilter() {
- super(new PrintWriter(System.out, true));
- }
-
- public void write(int ch)
- throws IOException {
- if (mAtLineStart) {
- out.write("# ");
- mAtLineStart = false;
- }
- out.write(ch);
- mAtLineStart = (ch == '\n');
- }
-
- public void write(String s, int off, int len)
- throws IOException {
- write(s.toCharArray(), off, len);
- }
-
- public void write(char[] a, int off, int len)
- throws IOException {
- for (int i = 0; i < len; i++) {
- write(a[off+i]);
- }
- }
-
- private boolean mAtLineStart = true;
- }
-
-
- private transient int mMaximumWorkerCount = 0;
- private transient String mLsfQueue = null;
- private transient String mLsfLogDirectory = null;
- private transient boolean mEnableGcLogging = false;
- private transient boolean mIsRunning = false;
- private transient int mWorkerId = 0;
- private transient int mProcessId = 0;
- private transient WorkerThread mWorkerThread = null;
- private transient String mAlgorithmName = null;
- private transient String mServerHost = null;
- private transient int mServerPort = 0;
- private transient DistributedCallService mDistributedCallService = null;
- private transient DistributedCallServer mDistributedCallServer = null;
-}
diff --git a/java/lib/edu/mit/broad/dcp/DistributedAlgorithmWorker.java b/java/lib/edu/mit/broad/dcp/DistributedAlgorithmWorker.java
deleted file mode 100644
index dcee13eb8..000000000
--- a/java/lib/edu/mit/broad/dcp/DistributedAlgorithmWorker.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2007 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-import java.util.*;
-
-/**
- * Command line driver for distributed worker invocation.
- */
-public class DistributedAlgorithmWorker
-{
- public static void main(String[] args)
- throws Exception {
- new DistributedAlgorithmWorker().run(args);
- }
-
- private void run(String[] args)
- throws Exception {
-
- if (!parseArguments(args)) {
- System.exit(1);
- }
- System.out.println("# DistributedAlgorithmWorker");
- System.out.println("# Started at " + new Date());
- runDistributedWorker();
- System.out.println("# Ended at " + new Date());
- }
-
- private boolean parseArguments(String[] args) {
-
- int argpos = 0;
- int argsleft = 0;
-
- while (argpos < args.length) {
- argsleft = args.length - argpos;
- String arg = args[argpos];
- if (arg.equals("-serverHost") && argsleft > 1) {
- argpos++;
- mServerHost = args[argpos++];
- } else if (arg.equals("-serverPort") && argsleft > 1) {
- argpos++;
- mServerPort = Integer.parseInt(args[argpos++]);
- } else if (arg.equals("-workerId") && argsleft > 1) {
- argpos++;
- mWorkerId = new Integer(args[argpos++]);
- } else if (arg.equals("-processId") && argsleft > 1) {
- argpos++;
- mProcessId = new Integer(args[argpos++]);
- } else if (arg.equals("-debug")) {
- argpos++;
- mDebug = true;
- continue;
- } else if (arg.equals("-verbose")) {
- argpos++;
- mVerbose = true;
- continue;
- } else if (arg.startsWith("-")) {
- usage();
- return false;
- } else {
- break;
- }
- }
-
- argsleft = args.length - argpos;
- if (argsleft != 0) {
- usage();
- return false;
- }
-
- return true;
- }
-
- private void usage() {
- System.out.println("Usage: DistributedWorkerMain ...");
- System.out.println(" -serverHost ");
- System.out.println(" -serverPort ");
- System.out.println(" -workerId ");
- System.out.println(" -processId ");
- System.out.println(" -verbose");
- System.out.println(" -debug");
- }
-
- private void runDistributedWorker()
- throws Exception {
-
- DistributedAlgorithm algorithm = null;
- String serverAddress = getServerHost() + ":" + getServerPort();
- try {
- String url = "rmi://" + serverAddress + "/DistributedCallService";
- DistributedCallService server =
- (DistributedCallService) java.rmi.Naming.lookup(url);
- algorithm = server.getAlgorithm();
- } catch (java.rmi.ConnectException exc) {
- System.out.println("# Server " + serverAddress + " not responding.");
- return;
- }
-
- algorithm.setServerHost(getServerHost());
- algorithm.setServerPort(getServerPort());
- algorithm.runWorker(getWorkerId(), getProcessId());
- }
-
- private Integer getWorkerId() {
- return mWorkerId;
- }
-
- private Integer getProcessId() {
- return mProcessId;
- }
-
- private String getServerHost() {
- return mServerHost;
- }
-
- private int getServerPort() {
- return mServerPort;
- }
-
-
- private boolean mDebug = false;
- private boolean mVerbose = false;
- private String mServerHost = null;
- private int mServerPort = 0;
- private Integer mWorkerId = null;
- private Integer mProcessId = null;
-}
diff --git a/java/lib/edu/mit/broad/dcp/DistributedCallServer.java b/java/lib/edu/mit/broad/dcp/DistributedCallServer.java
deleted file mode 100644
index 995eff571..000000000
--- a/java/lib/edu/mit/broad/dcp/DistributedCallServer.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2008 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-
-import edu.mit.broad.dcp.message.*;
-
-import java.rmi.server.UnicastRemoteObject;
-import java.util.*;
-
-public class DistributedCallServer
- extends UnicastRemoteObject
- implements DistributedCallService
-{
- public DistributedCallServer()
- throws java.rmi.RemoteException {
- }
-
- public void setAlgorithm(DistributedAlgorithm algorithm) {
- mAlgorithm = algorithm;
- }
-
- public DistributedAlgorithm getAlgorithm() {
- return mAlgorithm;
- }
-
- public long writeMessage(DistributedCallMessage message) {
- message.setCallStatus(CallStatus.PENDING);
- message.setCallId(generateCallId());
- if (message.getReceiverWorkerId().equals(0)) {
- synchronized (mMessageQueue) {
- mMessageQueue.addLast(message);
- }
- } else {
- synchronized (mMessageQueue) {
- mMessageQueue.addFirst(message);
- }
- }
- return message.getCallId();
- }
-
- public DistributedCallMessage acceptMessage(int workerId, int processId) {
- if (workerId <= 0) {
- throw new IllegalArgumentException("Invalid worker ID: " + workerId);
- }
- if (processId <= 0) {
- throw new IllegalArgumentException("Invalid process ID: " + processId);
- }
- synchronized (mMessageQueue) {
- Iterator iterator = mMessageQueue.iterator();
- while (iterator.hasNext()) {
- DistributedCallMessage message = iterator.next();
- if (message.getCallStatus() != CallStatus.PENDING) {
- continue;
- }
- int receiverId = message.getReceiverWorkerId();
- if (receiverId == workerId ||
- (receiverId == 0 && workerId > 1)) {
- message.setCallStatus(CallStatus.PROCESSING);
- message.setReceiverWorkerId(workerId);
- message.setReceiverProcessId(processId);
- return message;
- }
- }
- }
-
- return null;
- }
-
- public void completeMessage(int workerId, int processId, long callId) {
- if (workerId <= 0) {
- throw new IllegalArgumentException("Invalid worker ID: " + workerId);
- }
- if (processId <= 0) {
- throw new IllegalArgumentException("Invalid process ID: " + processId);
- }
- if (callId <= 0) {
- throw new IllegalArgumentException("Invalid call ID: " + callId);
- }
- synchronized (mMessageQueue) {
- Iterator iterator = mMessageQueue.iterator();
- while (iterator.hasNext()) {
- DistributedCallMessage message = iterator.next();
- if (message.getCallId().longValue() == callId) {
- if (message.getCallStatus() != CallStatus.PROCESSING) {
- throw new IllegalStateException("Call #" + callId + " not in state PROCESSING");
- }
- if (!message.getReceiverWorkerId().equals(workerId)) {
- throw new IllegalStateException("Call #" + callId + " assigned to worker " + message.getReceiverWorkerId() + " not worker " + workerId);
- }
- if (!message.getReceiverProcessId().equals(processId)) {
- throw new IllegalStateException("Call #" + callId + " assigned to process " + message.getReceiverProcessId() + " not process " + processId);
- }
- iterator.remove();
- return;
- }
- }
- }
-
- throw new IllegalArgumentException("Unrecognized call ID " + callId);
- }
-
- public boolean isQueueEmpty() {
- synchronized (mMessageQueue) {
- return mMessageQueue.isEmpty();
- }
- }
-
- public void stop() {
- try {
- UnicastRemoteObject.unexportObject(this, false);
- } catch (java.rmi.NoSuchObjectException exc) {
- throw new RuntimeException("Exception unexporting object: " + exc.getMessage(),
- exc);
- }
- }
-
- private synchronized long generateCallId() {
- return ++mCallIdGenerator;
- }
-
- private long mCallIdGenerator = 0;
- private DistributedAlgorithm mAlgorithm = null;
- private LinkedList mMessageQueue =
- new LinkedList();
-}
diff --git a/java/lib/edu/mit/broad/dcp/DistributedCallService.java b/java/lib/edu/mit/broad/dcp/DistributedCallService.java
deleted file mode 100644
index 202b25f42..000000000
--- a/java/lib/edu/mit/broad/dcp/DistributedCallService.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2008 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp;
-
-import edu.mit.broad.dcp.message.*;
-
-public interface DistributedCallService
- extends java.rmi.Remote
-{
- public DistributedAlgorithm getAlgorithm()
- throws java.rmi.RemoteException;
- public long writeMessage(DistributedCallMessage message)
- throws java.rmi.RemoteException;
- public DistributedCallMessage acceptMessage(int workerId, int processId)
- throws java.rmi.RemoteException;
- public void completeMessage(int workerId, int processId, long callId)
- throws java.rmi.RemoteException;
-}
diff --git a/java/lib/edu/mit/broad/dcp/message/DistributedCallMessage.java b/java/lib/edu/mit/broad/dcp/message/DistributedCallMessage.java
deleted file mode 100644
index 1b0fa0a4d..000000000
--- a/java/lib/edu/mit/broad/dcp/message/DistributedCallMessage.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2007 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp.message;
-
-import edu.mit.broad.dcp.CallStatus;
-
-public class DistributedCallMessage
- extends DistributedMessage
-{
- public DistributedCallMessage() {
- }
-
- public Long getCallId() {
- return mCallId;
- }
-
- public void setCallId(Long value) {
- mCallId = value;
- }
-
- public CallStatus getCallStatus() {
- return mCallStatus;
- }
-
- public void setCallStatus(CallStatus value) {
- mCallStatus = value;
- }
-
- public String getMethodName() {
- return mMethodName;
- }
-
- public void setMethodName(String value) {
- mMethodName = value;
- }
-
- public Object[] getMethodArgs() {
- return mMethodArgs;
- }
-
- public void setMethodArgs(Object[] value) {
- mMethodArgs = value;
- }
-
- public String toString() {
- StringBuilder builder = new StringBuilder();
- builder.append("DistributedCallMessage");
- builder.append("(");
- builder.append("" + getSenderWorkerId());
- builder.append(",");
- builder.append("" + getSenderProcessId());
- builder.append(",");
- builder.append("" + getReceiverWorkerId());
- builder.append(",");
- builder.append("" + getReceiverProcessId());
- builder.append(",");
- builder.append("" + mCallId);
- builder.append(",");
- builder.append("" + mCallStatus);
- builder.append(",");
- builder.append("" + mMethodName);
- builder.append(",");
- if (mMethodArgs == null) {
- builder.append("" + mMethodArgs);
- } else {
- builder.append("[");
- for (int i = 0; i < mMethodArgs.length; i++) {
- if (i > 0) {
- builder.append(",");
- }
- builder.append("" + mMethodArgs[i]);
- }
- builder.append("]");
- }
- builder.append(")");
- return builder.toString();
- }
-
- public Long mCallId;
- public CallStatus mCallStatus;
- public String mMethodName;
- public Object[] mMethodArgs;
-}
diff --git a/java/lib/edu/mit/broad/dcp/message/DistributedMessage.java b/java/lib/edu/mit/broad/dcp/message/DistributedMessage.java
deleted file mode 100644
index a5e837a69..000000000
--- a/java/lib/edu/mit/broad/dcp/message/DistributedMessage.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * The Broad Institute
- * SOFTWARE COPYRIGHT NOTICE AGREEMENT
- * This software and its documentation are copyright 2007 by the
- * Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
- *
- * This software is supplied without any warranty or guaranteed support whatsoever. Neither
- * the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
- */
-package edu.mit.broad.dcp.message;
-
-
-public class DistributedMessage
-{
- public DistributedMessage() {
- }
-
- public Integer getSenderWorkerId() {
- return mSenderWorkerId;
- }
-
- public void setSenderWorkerId(Integer value) {
- mSenderWorkerId = value;
- }
-
- public Integer getSenderProcessId() {
- return mSenderProcessId;
- }
-
- public void setSenderProcessId(Integer value) {
- mSenderProcessId = value;
- }
-
- public Integer getReceiverWorkerId() {
- return mReceiverWorkerId;
- }
-
- public void setReceiverWorkerId(Integer value) {
- mReceiverWorkerId = value;
- }
-
- public Integer getReceiverProcessId() {
- return mReceiverProcessId;
- }
-
- public void setReceiverProcessId(Integer value) {
- mReceiverProcessId = value;
- }
-
- public Integer mSenderWorkerId;
- public Integer mSenderProcessId;
- public Integer mReceiverWorkerId;
- public Integer mReceiverProcessId;
-}
diff --git a/java/lib/edu/mit/broad/picard/PicardException.java b/java/lib/edu/mit/broad/picard/PicardException.java
deleted file mode 100644
index 4e36ba648..000000000
--- a/java/lib/edu/mit/broad/picard/PicardException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard;
-
-/**
- * Basic Picard runtime exception that, for now, does nothing much
- *
- * @author Kathleen Tibbetts
- */
-public class PicardException extends RuntimeException
-{
- public PicardException(String message) {
- super(message);
- }
-
- public PicardException(String message, Throwable throwable) {
- super(message, throwable);
- }
-
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/AbstractBaseAligner.java b/java/lib/edu/mit/broad/picard/aligner/AbstractBaseAligner.java
deleted file mode 100644
index 54f0ab9aa..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/AbstractBaseAligner.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner;
-
-import edu.mit.broad.picard.io.IoUtil;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-import java.util.List;
-
-/**
- * Abstract base class for use by Aligner implementations. Provides a constructor and
- * accessors for common inputs and outputs.
- *
- * @author Kathleen Tibbetts
- */
-public abstract class AbstractBaseAligner implements Aligner {
-
- private final Stringency stringency; // The stringency of the alignment
- private final File readsBamFile; // The BAM file containing the read data
- private final String outputPrefix; // The directory and file name prefix for outputs
- private final String referenceFileDir; // The directory where the reference file can be found
- private final int clipPoints[]; // The clip points to use
- private final Integer expectedInsertSize; // Expected insert size; null for non-paired-end lanes
- private final Integer readsToAlign; // The number of reads to align (all if null)
- private final boolean pairedReads; // Whether this is a paired-end run
- private final int readLength;
- // Parameters specific to the Aligner implementation being used
- private final Map customParametersMap;
-
- /**
- * Constructor that sets every parameter.
- *
- * @param stringency the stringency of the alignment
- * @param readsBamFile the BAM file containing the reads
- * @param outputPrefix the directory and filename prefix for output
- * @param referenceFileDir the directory where the reference file is located
- * @param clipPoints the clip points
- * @param expectedInsertSize the expected insert size (null for non-PE lanes)
- * @param readsToAlign the number of reads to align
- * @param customParametersMap parameters specific to the Aligner implementation
- */
- public AbstractBaseAligner(Stringency stringency, File readsBamFile, String outputPrefix,
- String referenceFileDir, int clipPoints[], Integer expectedInsertSize,
- Integer readsToAlign, Map customParametersMap,
- boolean pairedReads, int readLength) {
-
- // First, a little validation
- if (clipPoints != null && clipPoints.length != 4) {
- throw new IllegalArgumentException("Length of clipPoints array argument must be 4.");
- }
- IoUtil.assertFileIsReadable(readsBamFile);
-
- this.stringency = stringency;
- this.readsBamFile = readsBamFile;
- this.outputPrefix = outputPrefix;
- this.referenceFileDir = referenceFileDir;
- this.clipPoints = clipPoints != null ? clipPoints : new int[4];
- this.expectedInsertSize = expectedInsertSize;
- this.readsToAlign = readsToAlign;
- this.customParametersMap = customParametersMap;
- this.pairedReads = pairedReads;
- this.readLength = readLength;
- }
-
- /**
- * Utility method for deleting a list of files, to be used by the
- * cleanup method of sub-classes
- *
- * @param files the list of files to delete
- */
- protected final void deleteFiles(List files) {
- for (File f : files) {
- f.delete();
- }
- }
-
- // Accessors
- protected final Stringency getStringency() { return stringency; }
- protected final File getReadsBamFile() { return readsBamFile; }
- protected final String getOutputPrefix() { return outputPrefix; }
- protected final String getReferenceFileDir() { return referenceFileDir; }
- protected final int[] getClipPoints() { return clipPoints; }
- protected final Integer getExpectedInsertSize() { return expectedInsertSize; }
- protected final Integer getReadsToAlign() { return readsToAlign; }
- protected final Map getCustomParametersMap() { return customParametersMap; }
- protected final boolean isPairedReads() { return pairedReads; }
- protected final int getReadLength() { return readLength; }
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/Aligner.java b/java/lib/edu/mit/broad/picard/aligner/Aligner.java
deleted file mode 100644
index d0fdf47de..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/Aligner.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner;
-
-/**
- * API for aligners. Clients must call these methods in order, as each depends on
- * the previous one, but they may call them multiple times and need not call them all.
- * This allows steps to be rerun and also lets the caller review intermediate files
- * when troubleshooting.
- *
- * @author Kathleen Tibbetts
- */
-public interface Aligner {
-
- public static enum Stringency{ low, high };
-
- /**
- * Prepares all the necessary inputs for the alignment process from a BAM file of read data.
- */
- public void prepareInputs();
-
- /**
- * Does the alignment and produces output in the underlying form of the aligner.
- */
- public void align();
-
- /**
- * Converts the output of the aligner to BAM format
- */
- public void prepareOutput();
-
- /**
- * Cleans up intermediate files (the files created in by and for the underlying aligner by the
- * prepareInputs() and align() methods. Does not clean up the original source files or the final BAM file.
- */
- public void cleanup();
-
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/BamToBfqWriter.java b/java/lib/edu/mit/broad/picard/aligner/maq/BamToBfqWriter.java
deleted file mode 100644
index 1f3cd55ac..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/BamToBfqWriter.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-import edu.mit.broad.sam.SAMFileReader;
-import edu.mit.broad.sam.SAMRecord;
-import edu.mit.broad.sam.util.BinaryCodec;
-import edu.mit.broad.picard.io.IoUtil;
-import edu.mit.broad.picard.PicardException;
-import edu.mit.broad.picard.filter.*;
-import edu.mit.broad.picard.util.PeekableIterator;
-import edu.mit.broad.picard.util.Log;
-import edu.mit.broad.picard.sam.ReservedTagConstants;
-
-import java.io.File;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Arrays;
-
-/**
- * Class to take unmapped reads in BAM file format and create Maq binary fastq format file(s) --
- * one or two of them, depending on whether it's a paired-end read. This relies on the unmapped
- * BAM file having all paired reads together in order.
- */
-public class BamToBfqWriter {
-
- private final File bamFile;
- private final String outputPrefix;
- private boolean pairedReads = false;
- private int wrote = 0;
- private int increment = 1;
- private int chunk = 0;
- private BinaryCodec codec1;
- private BinaryCodec codec2;
- private final Log log = Log.getInstance(BamToBfqWriter.class);
-
- /**
- * Constructor
- *
- * @param bamFile the BAM file to read from
- * @param outputPrefix the directory and file prefix for the binary fastq files
- * @param total the total number of records that should be written, drawn evenly
- * from throughout the file (null for all).
- * @param chunk the maximum number of records taht should be written to any one file
- * @param pairedReads whether these reads are from a paired-end run
- */
- public BamToBfqWriter(File bamFile, String outputPrefix, Integer total, Integer chunk, boolean pairedReads) {
- this.bamFile = bamFile;
- this.outputPrefix = outputPrefix;
- this.pairedReads = pairedReads;
- if (total != null) {
- double writeable = (double)countWritableRecords();
- this.increment = (int)Math.floor(writeable/total.doubleValue());
- }
- if (chunk != null) {
- this.chunk = chunk;
- }
- }
-
- /**
- * Constructor
- *
- * @param bamFile the BAM file to read from
- * @param outputPrefix the directory and file prefix for the binary fastq files
- * @param pairedReads whether these reads are from a paired-end run
- */
- public BamToBfqWriter(File bamFile, String outputPrefix, boolean pairedReads) {
- this(bamFile, outputPrefix, null, null, pairedReads);
- }
-
- /**
- * Writes the binary fastq file(s) to the output directory
- */
- public void writeBfqFiles() {
-
- Iterator iterator = (new SAMFileReader(IoUtil.openFileForReading(this.bamFile))).iterator();
-
- // Filter out noise reads and reads that fail the quality filter
- TagFilter tagFilter = new TagFilter(ReservedTagConstants.XN, 1);
- FailsVendorReadQualityFilter qualityFilter = new FailsVendorReadQualityFilter();
-
- if (!pairedReads) {
- writeSingleEndBfqs(iterator, Arrays.asList(tagFilter, qualityFilter));
- codec1.close();
- }
- else {
- writePairedEndBfqs(iterator, tagFilter, qualityFilter);
- codec1.close();
- codec2.close();
- }
- log.info("Wrote " + wrote + " bfq records.");
-
- }
-
- /**
- * Path for writing bfqs for paired-end reads
- *
- * @param iterator the iterator witht he SAM Records to write
- * @param tagFilter the filter for noise reads
- * @param qualityFilter the filter for PF reads
- */
- private void writePairedEndBfqs(Iterator iterator, TagFilter tagFilter,
- FailsVendorReadQualityFilter qualityFilter) {
- // Open the codecs for writing
- int fileIndex = 0;
- initializeNextBfqFiles(fileIndex++);
-
- int records = 0;
-
- while (iterator.hasNext()) {
- SAMRecord first = iterator.next();
- if (!iterator.hasNext()) {
- throw new PicardException("Mismatched number of records in " + this.bamFile.getAbsolutePath());
- }
- SAMRecord second = iterator.next();
- if (!second.getReadName().equals(first.getReadName()) ||
- first.getFirstOfPairFlag() == second.getFirstOfPairFlag()) {
- throw new PicardException("Unmatched read pairs in " + this.bamFile.getAbsolutePath() +
- ": " + first.getReadName() + ", " + second.getReadName() + ".");
- }
-
- // If both are noise reads, filter them out
- if (tagFilter.filterOut(first) && tagFilter.filterOut(second)) {
- // skip it
- }
- // If either fails to pass filter, then exclude them as well
- else if (qualityFilter.filterOut(first) || qualityFilter.filterOut(second)) {
- // skip it
- }
- // Otherwise, write them out
- else {
- records++;
- if (records % increment == 0) {
- first.setReadName(first.getReadName() + "#0/1");
- writeFastqRecord(first.getFirstOfPairFlag() ? codec1 : codec2, first);
- second.setReadName(second.getReadName() + "#0/2");
- writeFastqRecord(second.getFirstOfPairFlag() ? codec1 : codec2, second);
- wrote++;
- if (wrote % 1000000 == 0) {
- log.info(wrote + " records written.");
- }
- if (chunk > 0 && wrote % chunk == 0) {
- initializeNextBfqFiles(fileIndex++);
- }
- }
- }
- }
- }
-
- /**
- * Path for writing bfqs for single-end reads
- *
- * @param iterator the iterator witht he SAM Records to write
- * @param filters the list of filters to be applied
- */
- private void writeSingleEndBfqs(Iterator iterator, List filters) {
-
- // Open the codecs for writing
- int fileIndex = 0;
- initializeNextBfqFiles(fileIndex++);
-
- int records = 0;
-
- FilteringIterator it = new FilteringIterator(iterator, new AggregateFilter(filters));
- while (it.hasNext()) {
- SAMRecord record = it.next();
- records++;
- if (records % increment == 0) {
-
- writeFastqRecord(codec1, record);
- wrote++;
- if (wrote % 1000000 == 0) {
- log.info(wrote + " records processed.");
- }
- if (chunk > 0 && wrote % chunk == 0) {
- initializeNextBfqFiles(fileIndex++);
- }
- }
- }
- }
-
- /**
- * Closes any the open bfq file(s), if any, and opens the new one(s)
- *
- * @param fileIndex the index (counter) of the files to write
- */
- private void initializeNextBfqFiles(int fileIndex) {
- // Close the codecs if they were writing before
- if (codec1 != null) {
- codec1.close();
- if (pairedReads) {
- codec2.close();
- }
- }
-
- // Open new file, using the fileIndex.
- File bfq1 = getOutputFile(this.outputPrefix , 1, fileIndex);
- codec1 = new BinaryCodec(IoUtil.openFileForWriting(bfq1));
- log.info("Now writing to file " + bfq1.getAbsolutePath());
- if (pairedReads) {
- File bfq2 = getOutputFile(this.outputPrefix , 2, fileIndex);
- codec2 = new BinaryCodec(IoUtil.openFileForWriting(bfq2));
- log.info("Now writing to file " + bfq2.getAbsolutePath());
- }
- }
-
- /**
- * Writes out a SAMRecord in Maq fastq format
- *
- * @param codec the code to write to
- * @param rec the SAMRecord to write
- */
- private void writeFastqRecord(BinaryCodec codec, SAMRecord rec) {
-
- // Writes the length of the read name and then the name (null-terminated)
- codec.writeString(rec.getReadName(), true, true);
-
- char seqs[] = rec.getReadString().toCharArray();
- char quals[] = rec.getBaseQualityString().toCharArray();
-
- // Write the length of the sequence
- codec.writeInt(seqs.length);
-
- // Calculate and write the sequence and qualities
- byte seqsAndQuals[] = new byte[seqs.length];
-
- for (int i = 0; i < seqs.length; i++) {
- int quality = Math.min(quals[i]-33, 63);
- int base;
- switch(seqs[i]) {
- case 'A':
- case 'a':
- base = 0;
- break;
- case 'C':
- case 'c':
- base = 1;
- break;
- case 'G':
- case 'g':
- base = 2;
- break;
- case 'T':
- case 't':
- base = 3;
- break;
- case 'N':
- case 'n':
- case '.':
- base = 0;
- quality = 0;
- break;
- default:
- throw new PicardException("Unknown base when writing bfq file: " + seqs[i]);
- }
- seqsAndQuals[i] = (byte) (base << 6 | quality);
- }
- codec.writeBytes(seqsAndQuals);
- }
-
- private int countWritableRecords() {
- int count = 0;
- PeekableIterator it = new PeekableIterator((new SAMFileReader(IoUtil.openFileForReading(this.bamFile))).iterator());
- if (!this.pairedReads) {
- // Filter out noise reads and reads that fail the quality filter
- List filters = new ArrayList();
- filters.add(new TagFilter(ReservedTagConstants.XN, 1));
- filters.add(new FailsVendorReadQualityFilter());
- FilteringIterator itr = new FilteringIterator(it, new AggregateFilter(filters));
- while (itr.hasNext()) {
- itr.next();
- count++;
- }
- }
- else {
- while (it.hasNext()) {
- SAMRecord first = it.next();
- SAMRecord second = it.next();
- // If both are noise reads, filter them out
- if (first.getAttribute(ReservedTagConstants.XN) != null &&
- second.getAttribute(ReservedTagConstants.XN) != null) {
- // skip it
- }
- // If either fails to pass filter, then exclude them as well
- else if (first.getReadFailsVendorQualityCheckFlag() || second.getReadFailsVendorQualityCheckFlag() ) {
- // skip it
- }
- // Otherwise, write them out
- else {
- count++;
- }
- }
- }
- it.close();
- return count;
- }
-
- /**
- * Constructs the name for the output file and returns the file
- *
- * @param outputPrefix the directory and file prefix for the output bfq file
- * @param read whether this is the file for the first or second read
- * @return a new File object for the bfq file.
- */
- private File getOutputFile(String outputPrefix, int read, int index) {
- File result = new File(outputPrefix + "." + index + "." + read + ".bfq");
- IoUtil.assertFileIsWritable(result);
- return result;
- }
-
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/MapFileIterator.java b/java/lib/edu/mit/broad/picard/aligner/maq/MapFileIterator.java
deleted file mode 100644
index af5574185..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/MapFileIterator.java
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-import edu.mit.broad.sam.*;
-import edu.mit.broad.sam.util.CloseableIterator;
-import edu.mit.broad.sam.util.BinaryCodec;
-import edu.mit.broad.sam.util.StringUtil;
-import edu.mit.broad.picard.io.IoUtil;
-import edu.mit.broad.picard.PicardException;
-import edu.mit.broad.picard.util.SamPairUtil;
-
-import java.io.File;
-import java.io.BufferedInputStream;
-import java.util.*;
-
-/**
- * Reads a Maq map file and returns an an iterator of SAMRecords and a populated header
- *
- * IMPORTANT! Even though the reads in the map file are in coordinate order, this iterator
- * will not necessarily return them in that order. For paired reads, both will be
- * returned only after *both* records have been seen.
- *
- * @author Kathleen Tibbetts
- */
-public class MapFileIterator implements CloseableIterator {
-
- public static final int MATE_UNMAPPED_FLAG = 64;
- public static final int READ_UNMAPPED_FLAG = 192;
-
- private static final int READ_NAME_LENGTH = 36;
- private static final int MAP_FORMAT = -1;
- private static final int MAX_READ_LENGTH = 128;
-
- private static final byte ACGT[] = {'A', 'C', 'G', 'T'};
-
- public static final String PROGRAM_RECORD = "0";
-
- private long recordCount = 0L;
- private int recordsRead = 0;
- private BinaryCodec mapCodec;
- private final SAMFileHeader header;
- private final boolean pairedReads;
- private final boolean jumpingLibrary;
- private final List next = new ArrayList();
- private final Map pending = new HashMap();
- private final List mapFiles = new LinkedList();
-
- /**
- * Constructor. Opens the map file, reads the record count and header from it,
- * creates the SAMFileHeader, and queues up the first read
- *
- * @param mapFile The Maq map file to read
- * @param commandLine The command line used to invoke Maq (for the header)
- * @param pairedReads Whether this is a paired-end run
- */
- public MapFileIterator(String commandLine, boolean pairedReads, boolean jumpingLibrary, File... mapFile) {
- if (mapFile.length == 0) {
- throw new IllegalArgumentException("At least one map file must be provided.");
- }
- mapFiles.addAll(Arrays.asList(mapFile));
-
- this.pairedReads = pairedReads;
- this.jumpingLibrary = jumpingLibrary;
-
- header = new SAMFileHeader();
- header.setSortOrder(SAMFileHeader.SortOrder.coordinate);
- SAMProgramRecord program = new SAMProgramRecord(PROGRAM_RECORD);
- program.setProgramVersion(MaqConstants.getProgramVersion());
- program.setCommandLine(commandLine);
- header.addProgramRecord(program);
-
- queueNextMapFile();
- }
-
- /**
- * Queues up the next map file
- *
- * @return true if there's another map file to iterate over
- */
- private boolean queueNextMapFile() {
-
- // Close the old file
- if (mapCodec != null) {
- mapCodec.close();
- }
-
- // If there are no more map files, return fales
- if (mapFiles.size() == 0) {
- return false;
- }
-
- // Otherwise, open the next file and reset the recordsRead count
- mapCodec = new BinaryCodec(new BufferedInputStream(IoUtil.openFileForReading(mapFiles.remove(0))));
- int format = mapCodec.readInt();
- if (format != MAP_FORMAT) {
- mapCodec.close();
- throw new PicardException("Unrecognized Maq map file format: " + format);
- }
- recordsRead = 0;
-
-
- // Read the sequences out of the map file and set them on the header
- int sequenceCount = mapCodec.readInt();
- List sequences = new ArrayList();
- for (int i = 0; i < sequenceCount; i++) {
- int length = mapCodec.readInt();
- // Write the sequence name, trimming off the null terminator
- sequences.add(new SAMSequenceRecord(mapCodec.readString(length).substring(0, length-1)));
- }
- if (header.getSequences() == null || header.getSequences().size() == 0) {
- header.setSequences(sequences);
- }
- else {
- // TODO: Check that the sequences match and throw and exception if they don't
- }
- recordCount = mapCodec.readLong();
-
- readNext();
- return true;
- }
-
- /**
- * Closes the BinaryCodec reading the map file
- */
- public void close() {
- mapCodec.close();
- }
-
- /**
- * @return true if the iteration has more elements
- */
- public boolean hasNext() {
- return next.size() > 0;
- }
-
- /**
- * @return the next SAMRecord in the iteration
- * @throws NoSuchElementException if this is called when hasNext() returns false
- */
- public SAMRecord next() {
- if (!hasNext()) {
- throw new NoSuchElementException("No more elements in this iteration");
- }
- SAMRecord result = next.remove(0);
- readNext();
- return result;
- }
-
- /**
- * Reads the next element from the map file. If we are done with it, we put it in the next
- * list; if we are waiting to see its mate, we put it in the pending map. Calls itself
- * repeatedly until there is at least one element in next.
- */
- private void readNext() {
-
- // If there's already a record queued up, just return
- if (next.size() > 0) {
- return;
- }
-
- // If we've read all there is, then any remaining records in the pending map should be returned.
- // If this is not a PE run, then the pending map will be empty and we're done.
- if (recordsRead == recordCount) {
- if (pending.size() > 0) {
- StringBuffer sb = new StringBuffer();
- for (String item : pending.keySet()) {
- sb.append(item).append("\n");
- }
- throw new PicardException("MapFileIterator pending map should have been empty but contained " +
- "the following records: " + sb.toString());
- }
- queueNextMapFile();
- return;
- }
-
- // Otherwise, we read until there is at least one record in the next list
- readMapRecord();
- if (next.size() == 0) {
- readNext();
- }
- }
-
- /**
- * Reads one record from the map file and throws it onto the pending map or the next list,
- * depending on whether we have already seen its mate
- */
- private void readMapRecord() {
-
- // Now that we've got all the data from the binary file, write a SAMRecord and add it to
- // the new BAM file
- SAMRecord record = new SAMRecord();
- record.setAttribute(SAMTag.PG.toString(), PROGRAM_RECORD);
- record.setReadPairedFlag(this.pairedReads);
-
- // the last base is the single-end mapping quality.
- byte seqsAndQuals[] = new byte[MAX_READ_LENGTH-1];
- mapCodec.readBytes(seqsAndQuals);
-
- byte singleEndMappingQualityOrIndelLength = mapCodec.readByte();
-
- // the length of the read
- int readLength = mapCodec.readUByte();
- setSeqsAndQuals(seqsAndQuals, readLength, record);
-
- // the final mapping quality (unless flag below is 130, then it is the
- // position of the indel (or 0 if no indel)
- int mappingQuality = mapCodec.readUByte();
-
- // mismatches in the 28bp (higher 4 bits) and mismatches (lower 4 bits)
- mapCodec.readUByte();
- // sum of errors of the best hit
- mapCodec.readUByte();
- // counts of all 0- and 1-mismatch hits on the reference
- mapCodec.readUByte();
- mapCodec.readUByte();
-
- // A bitwise flag. See the Maq docs for its full meaning
- int flag = mapCodec.readUByte();
-
- // the lower mapQ of the two ends (equals map_qual if unpaired); if flag is 130: mapQ of its mate
- int altQual = mapCodec.readUByte();
-
- // Index of the sequence for this read
- record.setReferenceIndex((int)mapCodec.readUInt(), getHeader());
-
- // Start position and strand
- long pos = mapCodec.readUInt();
- int startPos = ((int)((pos>>1)& 0x7FFFFFFF)) + 1;
- record.setAlignmentStart(startPos);
- record.setReadNegativeStrandFlag((pos&1) == 1);
-
- // offset of the mate (zero if unpaired, or two ends mapped to different chr)
- mapCodec.readInt();
-
- // The read name
- byte nameBytes[] = new byte[READ_NAME_LENGTH];
- mapCodec.readBytes(nameBytes);
- String name = StringUtil.bytesToString(nameBytes).trim();
- if (this.pairedReads) {
- if (name.endsWith("/1")) {
- record.setFirstOfPairFlag(true);
- record.setSecondOfPairFlag(false);
- }
- else if (name.endsWith("/2")) {
- record.setFirstOfPairFlag(false);
- record.setSecondOfPairFlag(true);
- }
- else {
- throw new PicardException("Unrecognized ending for paired read name: " + name);
- }
- name = name.substring(0, name.length()-2);
- }
- record.setReadName(name);
-
-
- if (flag != 130 || singleEndMappingQualityOrIndelLength == 0) { // No indel
- record.setCigarString(readLength + "M");
- record.setMappingQuality(mappingQuality);
- }
- else { // Indel
- int indelPos = mappingQuality;
- String cigar = indelPos + "M" + Math.abs(singleEndMappingQualityOrIndelLength);
- int remaining = readLength - indelPos;
- if (singleEndMappingQualityOrIndelLength > 0) {
- cigar += "I" + (remaining - singleEndMappingQualityOrIndelLength) + "M";
- }
- else {
- cigar += "D" + remaining + "M";
- }
- record.setCigarString(cigar);
- // In the docs, it look like there is a mapping quality for the mate, do we use that?
- record.setMappingQuality(altQual);
- }
-
- if (!pairedReads) {
- record.setProperPairFlag(false);
- next.add(record);
- }
- else {
- record.setMateUnmappedFlag(flag == MATE_UNMAPPED_FLAG);
- SAMRecord mate = pending.remove(record.getReadName());
-
- if (mate != null) {
- boolean proper = SamPairUtil.isProperPair(record, mate, jumpingLibrary);
- record.setProperPairFlag(proper);
- mate.setProperPairFlag(proper);
-
- SamPairUtil.setMateInfo(record, mate);
-
- int insertSize = SamPairUtil.computeInsertSize(record, mate);
- record.setInferredInsertSize(insertSize);
- mate.setInferredInsertSize(insertSize);
-
- if (!mate.getMateUnmappedFlag()) {
- next.add(record);
- }
- if (!record.getMateUnmappedFlag()) {
- next.add(mate);
- }
- }
- else {
- pending.put(record.getReadName(), record);
- }
- }
-
- // TODO: Figure out what do do about noise reads long-term
- // Note that it is possible that we have lost a "Noise read" annotation at this point. Since
- // we try to map a pair if only one of the reads is classified as "noise", then for any paired
- // reads where one was a noise read and one was not, we will lose the noise annotation on the
- // one noisy read. We have discussed either re-doing the noise evaluation here, modifying the
- // read name to carry the noise flag through Maq, or changing what reads we give to Maq.
-
- recordsRead++;
-
- }
-
- /**
- * Decodes the sequence and the qualities and sets them on the SAMrecords
- *
- * @param seqsAndQuals the list of seqs and quals
- * @param readLength the length of the read
- * @param sam the SAMRecord to populate
- */
- private void setSeqsAndQuals(byte seqsAndQuals[], int readLength, SAMRecord sam) {
- byte sequence[] = new byte[readLength];
- byte qualities[] = new byte[readLength];
- for (int i = 0; i < readLength; i++) {
- byte b = seqsAndQuals[i];
- qualities[i] = (byte)(b & 0x3F);
- if (b == 0) {
- sequence[i] = 'N';
- }
- else {
- sequence[i] = ACGT[(seqsAndQuals[i] >> 6) & 3];
- }
- }
- sam.setReadBases(sequence);
- sam.setBaseQualities(qualities);
- }
-
- /**
- * @throws UnsupportedOperationException -- not implemented
- */
- public void remove() {
- throw new UnsupportedOperationException("remove() not supported in MapFileIterator");
- }
-
- public SAMFileHeader getHeader() { return header; }
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/MaqAligner.java b/java/lib/edu/mit/broad/picard/aligner/maq/MaqAligner.java
deleted file mode 100644
index 6c1890818..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/MaqAligner.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-import edu.mit.broad.picard.aligner.Aligner;
-import edu.mit.broad.picard.aligner.AbstractBaseAligner;
-import edu.mit.broad.picard.PicardException;
-import edu.mit.broad.picard.util.Log;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.util.*;
-
-/**
- * Maq implementation of the Aligner interface
- */
-public class MaqAligner extends AbstractBaseAligner implements Aligner {
-
- // Constants related to Maq output files
- public static final String MAQ_MAP_SUFFIX = ".out.aln.map";
- public static final String MAQ_LOG_SUFFIX = ".out.map.log";
-
- // Internal constant for multi-plexing lane data
- private static final int READ_CHUNK_SIZE = 2000000;
-
- public static final String REFERENCE_FILE_SUFFIX = ".bfa";
-
- private final Log log = Log.getInstance(MaqAligner.class);
-
- private String commandLine = null;
-
-
- /**
- * Constructor that sets every parameter. All other constructors delegate to this one.
- *
- * @param stringency the stringency of the alignment
- * @param readsBamFile the BAM file containing the reads
- * @param outputPrefix the directory and filename prefix for output
- * @param referenceFileDir the directory where the reference file is located
- * @param clipPoints the clip points
- * @param expectedInsertSize the expected insert size (null for non-PE lanes)
- * @param readsToAlign the number of reads to align
- * @param customParametersMap parameters specific to the Aligner implementation
- */
- public MaqAligner(Stringency stringency, File readsBamFile, String outputPrefix,
- String referenceFileDir, int clipPoints[], Integer expectedInsertSize,
- Integer readsToAlign, Map customParametersMap,
- boolean pairedReads, int readLength) {
-
- super(stringency, readsBamFile, outputPrefix, referenceFileDir, clipPoints,
- expectedInsertSize, readsToAlign, customParametersMap, pairedReads, readLength);
- }
-
- /**
- * Prepares all the necessary inputs for the alignment process from a BAM file of read data.
- */
- public void prepareInputs() {
- log.info("Preparing Maq inputs.");
- BamToBfqWriter writer = new BamToBfqWriter(this.getReadsBamFile(), this.getOutputPrefix(),
- this.getReadsToAlign(), READ_CHUNK_SIZE, isPairedReads());
- writer.writeBfqFiles();
- }
-
- /**
- * Does the alignment and produces output in the underlying form of the aligner.
- */
- public void align() {
- log.info("Running Maq alignment.");
-
- // Temporary hack until we get the multi-tasking code from Seva
- List mapFileNames = new ArrayList(); // All map files that we will merge together at the end
-
- String maqParams = MaqConstants.SWITCH_RANDOM_SEED + " " + MaqConstants.DEFAULT_RANDOM_SEED;
-
- if (this.getStringency() == Stringency.high) {
- maqParams += " " + MaqConstants.SWITCH_MAX_OUTER_DISTANCE + " " + Math.round(
- this.getExpectedInsertSize() * MaqConstants.HIGH_STRINGENCY_MAX_OUTER_DISTANCE_MULTIPLIER);
- maqParams += " " + MaqConstants.SWITCH_SUM_MISMATCHES + " " +
- MaqConstants.HIGH_STRINGENCY_SUM_MISMATCHES;
- }
- else {
- maqParams += " " + MaqConstants.SWITCH_MAX_OUTER_DISTANCE + " " +
- MaqConstants.LOW_STRINGENCY_MAX_OUTER_DISTANCE;
- // For low stringency, get at least 30 bases and then let half of what's remaining mismatch
- int maxMisMatches = (this.getReadLength() - 30)/2;
- maqParams += " " + MaqConstants.SWITCH_SUM_MISMATCHES + " " +
- (maxMisMatches * MaqConstants.LOW_STRINGENCY_QUALITY_FOR_MISMATCHES);
- }
-
- String referenceFile = new File(this.getReferenceFileDir()).listFiles(new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.endsWith(REFERENCE_FILE_SUFFIX);
- }
- })[0].getAbsolutePath();
-
- ProcessBuilder builder;
-
- // Map the bfq files, individually or in pairs
- SortedSet bfqs = new TreeSet(this.getBfqFiles());
- for (Iterator it = bfqs.iterator(); it.hasNext();) {
-
- String read1bfq = it.next().getAbsolutePath();
- String read2bfq = (this.isPairedReads()) ? it.next().getAbsolutePath() : "";
-
- String outputFileBase = read1bfq.substring(0, read1bfq.lastIndexOf('.')-2);
- String mapFile = outputFileBase + MAQ_MAP_SUFFIX;
- String logFile = outputFileBase + MAQ_LOG_SUFFIX;
-
- String command = MaqConstants.MAQ_HOME + MaqConstants.MAQ_COMMAND + " " + MaqConstants.MAP_COMMAND +
- " " + maqParams + " " + mapFile + " " + referenceFile + " " + read1bfq + " " + read2bfq +
- " 2> " + logFile;
- setCommandLine(getCommandLine() == null ? command : getCommandLine() + ";" + command);
- log.info("Executing command: " + command);
- try {
- builder = new ProcessBuilder(command.split(" "));
- Process p = builder.start();
- p.waitFor();
- }
- catch (Exception e) {
- throw new PicardException("Error starting Maq process", e);
- }
-
- mapFileNames.add(mapFile);
- }
-
- // If there's more than one map file, then merge them.
- String finalFileName = this.getOutputPrefix() + "." + this.getStringency() + MAQ_MAP_SUFFIX;
- if (mapFileNames.size() > 1) {
- String command = MaqConstants.MAQ_HOME + MaqConstants.MAQ_COMMAND + " " +
- MaqConstants.MERGE_COMMAND + " " + finalFileName;
- for (String name : mapFileNames) {
- command += " " + name;
- }
- setCommandLine(getCommandLine() == null ? command : getCommandLine() + ";" + command);
- log.info("Executing command: " + command);
-
- try {
- builder = new ProcessBuilder(command.split(" "));
- Process p = builder.start();
- p.waitFor();
- }
- catch (Exception e) {
- throw new PicardException("Error starting Maq process", e);
- }
- }
- else { // Otherwise rename the single map file so we can find it later
- File f = new File(mapFileNames.get(0));
- if (!f.renameTo(new File(finalFileName))) {
- throw new PicardException("Error renaming " + f.getAbsolutePath() + " to " + finalFileName);
- }
- }
- }
-
- /**
- * Converts the output of the aligner to BAM format
- */
- public void prepareOutput() {
- log.info("Preparing output from Maq alignment.");
- // TODO: MaqToBam
- }
-
- /**
- * Cleans up intermediate files (the files created in by and for the underlying aligner by the
- * prepareInputs() and align() methods. Does not clean up the original source files or the final BAM file.
- */
- public void cleanup() {
- log.info("Cleaning up Maq intermediate files.");
- this.deleteFiles(getBfqFiles());
-// this.deleteFiles(getMaqAlignmentFiles());
- }
-
- /**
- * Returns a list of zero to two BFQ files, depending on whether they are there
- * and whether it was a paired-end run or not
- *
- * @return a list of BFQ files
- */
- private List getBfqFiles() {
- File dir = new File(this.getOutputPrefix().substring(0, this.getOutputPrefix().lastIndexOf("/")));
- return Arrays.asList(dir.listFiles(new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.endsWith(".bfq");
- }
- }));
- }
-
- /**
- * Returns the Maq map files
- *
- * @return a list of Maq .map files
- */
- private List getMaqAlignmentFiles() {
- File dir = new File(this.getOutputPrefix().substring(0, this.getOutputPrefix().lastIndexOf("/")));
- return Arrays.asList(dir.listFiles(new FilenameFilter() {
- public boolean accept(File dir, String name) {
- // TODO: Add the text files if we do not read the binary map files
- return name.endsWith(MAQ_MAP_SUFFIX) || name.endsWith(MAQ_LOG_SUFFIX);
- }
- }));
- }
-
- public String getCommandLine() { return commandLine; }
- public void setCommandLine(String commandLine) { this.commandLine = commandLine; }
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/MaqConstants.java b/java/lib/edu/mit/broad/picard/aligner/maq/MaqConstants.java
deleted file mode 100644
index b5e4b9b59..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/MaqConstants.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-/**
- * Utility class to hold Maq-related constants (program name, location, switches, etc)
- */
-public class MaqConstants {
- // General Maq constants
- public static final String PROGRAM_NAME = "Maq";
- public static final String PROGRAM_VERSION = "0.7.1";
- public static final String MAQ_HOME = "/seq/dirseq/maq-0.7.1/";
-
- // Command-related constants
- public static final String MAQ_COMMAND = "maq";
- public static final String MAP_COMMAND = "map";
- public static final String MERGE_COMMAND = "mapmerge";
-
- // Constants related to Maq map switches
- public static final String SWITCH_SUM_MISMATCHES = "-e";
- public static final int HIGH_STRINGENCY_SUM_MISMATCHES = 100;
- public static final int LOW_STRINGENCY_QUALITY_FOR_MISMATCHES = 30;
-
- public static final String SWITCH_MAX_OUTER_DISTANCE = "-a";
- public static final int LOW_STRINGENCY_MAX_OUTER_DISTANCE = 1500;
- public static final double HIGH_STRINGENCY_MAX_OUTER_DISTANCE_MULTIPLIER = 1.5d;
-
- public static final String SWITCH_RANDOM_SEED = "-s";
- public static final int DEFAULT_RANDOM_SEED = 0;
-
- public static String getProgramVersion() { return PROGRAM_VERSION; }
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/MaqMapMerger.java b/java/lib/edu/mit/broad/picard/aligner/maq/MaqMapMerger.java
deleted file mode 100644
index 3b82cc106..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/MaqMapMerger.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-import edu.mit.broad.picard.io.IoUtil;
-import edu.mit.broad.picard.util.StringSortingCollectionFactory;
-import edu.mit.broad.picard.util.Log;
-import edu.mit.broad.picard.PicardException;
-import edu.mit.broad.sam.util.SortingCollection;
-import edu.mit.broad.sam.util.BinaryCodec;
-import edu.mit.broad.sam.util.CloseableIterator;
-import edu.mit.broad.sam.*;
-
-import java.io.File;
-import java.io.BufferedInputStream;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.nio.ByteBuffer;
-
-/**
- * Class to write a BAM file that includes the results from a Maq .map file along with the unaligned
- * reads from the original BAM file.
- *
- * Information on the meaning of the elements of the map file is drawn from the Maq documentation
- * on this page: http://maq.sourceforge.net/maqmap_format.shtml
- */
-public class MaqMapMerger {
-
- private final File mapFile;
- private final File sourceBamFile;
- private final File targetBamFile;
- private final boolean pairedReads;
- private final Log log = Log.getInstance(MaqMapMerger.class);
- private String commandLine = null;
- private List sequences = new ArrayList();
-
-
- /**
- * Constructor
- *
- * @param mapFile The Maq map file to parse
- * @param sourceBamFile The BAM file that was used as the input to the Maq aligner, which will
- * include info on all the reads that did not map
- * @param targetBamFile The file to which to write the merged
- */
- public MaqMapMerger(File mapFile, File sourceBamFile, File targetBamFile, boolean pairedReads) {
- IoUtil.assertFileIsReadable(mapFile);
- IoUtil.assertFileIsReadable(sourceBamFile);
- IoUtil.assertFileIsWritable(targetBamFile);
- this.mapFile = mapFile;
- this.sourceBamFile = sourceBamFile;
- this.targetBamFile = targetBamFile;
- this.pairedReads = pairedReads;
- }
-
- /**
- * Merges the alignment from the map file with the remaining records from the source BAM file.
- */
- public void mergeAlignment() {
- log.info("Processing map file: " + mapFile.getAbsolutePath());
- // Write the header
- MapFileIterator it = new MapFileIterator(getCommandLine(), this.pairedReads, false, this.mapFile);
- SAMFileHeader header = it.getHeader();
- SAMFileWriter writer = new SAMFileWriterFactory().makeBAMWriter(header, false, targetBamFile);
-
- // Write the alignments
- SortingCollection readNames = writeAlignments(it, writer);
-
- // We're done with the map file, so close it
- it.close();
- writeUnalignedReads(writer, readNames.iterator());
-
- // Now close the writer
- writer.close();
- }
-
-
- private void writeUnalignedReads(SAMFileWriter writer, CloseableIterator nameIterator) {
-
- int skipCount = 0;
- SAMFileReader reader = new SAMFileReader(IoUtil.openFileForReading(this.sourceBamFile));
- CloseableIterator bamRecords = reader.iterator();
-
- String readName = nameIterator.hasNext() ? nameIterator.next() : null;
- while(bamRecords.hasNext()) {
- SAMRecord rec = bamRecords.next();
- if (rec.getReadName().equals(readName)) {
- // skip it and pull the next name off the name iterator
- readName = nameIterator.hasNext() ? nameIterator.next() : null;
- skipCount++;
- }
- else {
- writer.addAlignment(rec);
- }
- }
-System.out.println("Skipped " + skipCount + " already-aligned records.");
- bamRecords.close();
- nameIterator.close();
- }
-
- private SortingCollection writeAlignments(MapFileIterator iterator, SAMFileWriter writer) {
-
-int wrote = 0;
- SortingCollection readNames = StringSortingCollectionFactory.newCollection();
- while (iterator.hasNext()) {
- SAMRecord record = iterator.next();
- readNames.add(record.getReadName());
- writer.addAlignment(record);
-wrote++;
- }
-System.out.println("Wrote " + wrote + " alignment records.");
- return readNames;
- }
-
- public void setCommandLine(String commandLine) { this.commandLine = commandLine; }
- public String getCommandLine() { return this.commandLine; }
-}
diff --git a/java/lib/edu/mit/broad/picard/aligner/maq/RunMaq.java b/java/lib/edu/mit/broad/picard/aligner/maq/RunMaq.java
deleted file mode 100644
index bc3741b02..000000000
--- a/java/lib/edu/mit/broad/picard/aligner/maq/RunMaq.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2008 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.aligner.maq;
-
-import edu.mit.broad.picard.cmdline.CommandLineProgram;
-import edu.mit.broad.picard.cmdline.Usage;
-import edu.mit.broad.picard.cmdline.Option;
-import edu.mit.broad.picard.aligner.Aligner;
-
-import java.io.File;
-import java.util.Map;
-import java.util.List;
-import java.util.HashMap;
-import java.util.ArrayList;
-
-/**
- * CommandLineProgram to generate to invoke BustardToBamWriter
- *
- * @author Kathleen Tibbetts
- */
-public class RunMaq extends CommandLineProgram {
- private static final String PROGRAM_VERSION = "1.0";
-
- // The following attributes define the command-line arguments
- @Usage
- public String USAGE =
- "Usage: " + getClass().getName() + " [options]\n\n" +
- "Invoke the Maq aligner.\n" +
- "Version: " + PROGRAM_VERSION +"\n";
-
- @Option(shortName="I", doc="The BAM file to parse.", optional=true)
- public File INPUT;
- @Option(shortName="O", doc="The directory and file prefix for all output.", optional=false)
- public String OUTPUT;
- @Option(shortName="L", doc="The read length.", optional=false)
- public Integer READ_LENGTH;
- @Option(shortName="S", doc="Stringency of the alignment.", optional=true)
- public Aligner.Stringency STRINGENCY;
- @Option(shortName="R", doc="Directory where the reference file is located.", optional=true)
- public String REFERENCE;
- @Option(shortName="C", doc="Clip points for the alignment.", optional=true, minElements=0, maxElements=4)
- public List CLIP_POINT = new ArrayList();
- @Option(shortName="E", doc="Expected insert size.", optional=true)
- public Integer EXPECTED_INSERT_SIZE;
- @Option(doc="Whether this is a paired-end run.", optional=false)
- public Boolean PE;
- @Option(shortName="NUM", doc="Number of reads to align (null = all).", optional=true)
- public Integer READS_TO_ALIGN;
- @Option(shortName="CUSTOM", doc="Custom parameter in the form name=value.", optional=true)
- public List CUSTOM_PARAMETER = new ArrayList();
- @Option(shortName="PREP", doc="Whether to prepare inputs for the alignement.", optional=true)
- public Boolean PREPARE = true;
- @Option(doc="Whether to do the alignement.", optional=true)
- public Boolean ALIGN = true;
- @Option(shortName="BAM", doc="Whether to generate a BAM file from the alignment output.", optional=true)
- public Boolean BAM_OUTPUT = true;
- @Option(doc="Whether to clean up intermediate input and output.", optional=true)
- public Boolean CLEANUP = true;
-
- protected int doWork() {
- int clipPoints[] = null;
- if (CLIP_POINT != null) {
- clipPoints = new int[4];
- int index=0;
- for (Integer i : CLIP_POINT) {
- clipPoints[index++] = i;
- }
- }
- Map params = null;
- if (CUSTOM_PARAMETER != null) {
- params = new HashMap();
- for (String param : CUSTOM_PARAMETER) {
- String nameAndVal[] = param.split("=");
- params.put(nameAndVal[0], nameAndVal[1]);
- }
- }
- Aligner aligner = new MaqAligner(STRINGENCY, INPUT, OUTPUT, REFERENCE, clipPoints,
- EXPECTED_INSERT_SIZE, READS_TO_ALIGN, params, PE, READ_LENGTH);
- if (PREPARE) {
- aligner.prepareInputs();
- }
- if (ALIGN) {
- aligner.align();
- }
- if (BAM_OUTPUT) {
- aligner.prepareOutput();
- }
- if (CLEANUP) {
- aligner.cleanup();
- }
- return 0;
- }
-
- /**
- * This is kind of a mess. Almost everything is optional, since you don't have to do all of the steps in the
- * alignement.
- * @return
- */
- protected boolean customCommandLineValidation() {
- if (PREPARE) {
- if( INPUT == null) {
- System.err.println("ERROR: INPUT must be specified when preparing inputs for the alignment.");
- return false;
- }
- if (CLIP_POINT.size() != 0 && CLIP_POINT.size() != 4) {
- System.err.println("ERROR: You must supply either 0 or 4 values for CLIP_POINT: " + CLIP_POINT.size());
- return false;
- }
- }
- if (ALIGN) {
- if (STRINGENCY == null) {
- System.err.println("ERROR: STRINGENCY must be specified when doing an alignment.");
- return false;
- }
- if (REFERENCE == null) {
- System.err.println("ERROR: REFERENCE must be specified when doing an alignment.");
- return false;
- }
- }
- return true;
- }
-
- public static void main(String[] argv) {
- System.exit(new RunMaq().instanceMain(argv));
- }
-}
diff --git a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParseException.java b/java/lib/edu/mit/broad/picard/cmdline/CommandLineParseException.java
deleted file mode 100644
index cfe74bbcc..000000000
--- a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParseException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2008 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.cmdline;
-
-public class CommandLineParseException extends RuntimeException{
- public CommandLineParseException() {
- }
-
- public CommandLineParseException(String s) {
- super(s);
- }
-
- public CommandLineParseException(String s, Throwable throwable) {
- super(s, throwable);
- }
-
- public CommandLineParseException(Throwable throwable) {
- super(throwable);
- }
-}
diff --git a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParser.java b/java/lib/edu/mit/broad/picard/cmdline/CommandLineParser.java
deleted file mode 100644
index 69b681abb..000000000
--- a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParser.java
+++ /dev/null
@@ -1,638 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2008 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.cmdline;
-
-import java.io.*;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.Field;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
-import java.util.*;
-
-import edu.mit.broad.picard.util.StringUtil;
-import edu.mit.broad.picard.PicardException;
-
-/**
- * Annotation-driven utility for parsing command-line arguments, checking for errors, and producing usage message.
- *
- * This class supports options of the form KEY=VALUE, plus positional arguments. Positional arguments must not contain
- * an equal sign lest they be mistaken for a KEY=VALUE pair.
- *
- * The caller must supply an object that both defines the command line and has the parsed options set into it.
- * For each possible KEY=VALUE option, there must be a public data member annotated with @Option. The KEY name is
- * the name of the data member. An abbreviated name may also be specified with the shortName attribute of @Option.
- * If the data member is a List, then the option may be specified multiple times. The type of the data member,
- * or the type of the List element must either have a ctor T(String), or must be an Enum. List options must
- * be initialized by the caller with some kind of list. Any other option that is non-null is assumed to have the given
- * value as a default. If an option has no default value, and does not have the optional attribute of @Option set,
- * is required. For List options, minimum and maximum number of elements may be specified in the @Option annotation.
- *
- * A single List data member may be annotated with the @PositionalArguments. This behaves similarly to a Option
- * with List data member: the caller must initialize the data member, the type must be constructable from String, and
- * min and max number of elements may be specified. If no @PositionalArguments annotation appears in the object,
- * then it is an error for the command line to contain positional arguments.
- *
- * A single String public data member may be annotated with @Usage. This string, if present, is used to
- * construct the usage message. Details about the possible options are automatically appended to this string.
- * If @Usage does not appear, a boilerplate usage message is used.
- */
-public class CommandLineParser {
- // For formatting option section of usage message.
- private static final int OPTION_COLUMN_WIDTH = 30;
- private static final int DESCRIPTION_COLUMN_WIDTH = 50;
-
- private static final Boolean[] TRUE_FALSE_VALUES = {Boolean.TRUE, Boolean.FALSE};
-
- // Use these if no @Usage annotation
- private static final String defaultUsagePreamble = "Usage: program [options...]\n";
- private static final String defaultUsagePreambleWithPositionalArguments =
- "Usage: program [options...] [positional-arguments...]\n";
- private static final String OPTIONS_FILE = "OPTIONS_FILE";
-
- /**
- * A typical command line program will call this to get the beginning of the usage message,
- * and then append a description of the program, like this:
- *
- * \@Usage(programVersion=PROGRAM_VERSION)
- * public String USAGE = CommandLineParser.getStandardUsagePreamble(getClass()) + "Frobnicates the freebozzle."
- */
- public static String getStandardUsagePreamble(Class mainClass) {
- return "USAGE: " + mainClass.getName() + " [options]\n\n";
- }
-
- // This is the object that the caller has provided that contains annotations,
- // and into which the values will be assigned.
- private final Object callerOptions;
-
- private String usagePreamble;
- // null if no @PositionalArguments annotation
- private Field positionalArguments;
- private int minPositionalArguments;
- private int maxPositionalArguments;
-
- // List of all the data members with @Option annotation
- private final List optionDefinitions = new ArrayList();
-
- // Maps long name, and short name, if present, to an option definition that is
- // also in the optionDefinitions list.
- private final Map optionMap = new HashMap();
-
- // For printing error messages when parsing command line.
- private PrintStream messageStream;
-
- // In case implementation wants to get at arg for some reason.
- private String[] argv;
-
-
- /**
- * This attribute is here just to facilitate printing usage for OPTIONS_FILE
- */
- public File IGNORE_THIS_PROPERTY;
-
- /**
- * Prepare for parsing command line arguments, by validating annotations.
- * @param callerOptions This object contains annotations that define the acceptable command-line options,
- * and ultimately will receive the settings when a command line is parsed.
- */
- public CommandLineParser(final Object callerOptions) {
- this.callerOptions = callerOptions;
-
- for (final Field field : this.callerOptions.getClass().getFields()) {
- if (field.getAnnotation(PositionalArguments.class) != null) {
- handlePositionalArgumentAnnotation(field);
- }
- if (field.getAnnotation(Usage.class) != null) {
- handleUsageAnnotation(field);
- }
- if (field.getAnnotation(Option.class) != null) {
- handleOptionAnnotation(field);
- }
- }
-
- if (usagePreamble == null) {
- if (positionalArguments == null) {
- usagePreamble = defaultUsagePreamble;
- } else {
- usagePreamble = defaultUsagePreambleWithPositionalArguments;
- }
- }
- }
-
- /**
- * Print a usage message based on the options object passed to the ctor.
- * @param stream Where to write the usage message.
- */
- public void usage(final PrintStream stream) {
- stream.print(usagePreamble);
- if (!optionDefinitions.isEmpty()) {
- stream.println("\nOptions:\n");
- for (final OptionDefinition optionDefinition : optionDefinitions) {
- printOptionUsage(stream, optionDefinition);
- }
- }
- final Field fileField;
- try {
- fileField = getClass().getField("IGNORE_THIS_PROPERTY");
- } catch (NoSuchFieldException e) {
- throw new PicardException("Should never happen", e);
- }
- final OptionDefinition optionsFileOptionDefinition =
- new OptionDefinition(fileField, OPTIONS_FILE, "",
- "File of OPTION_NAME=value pairs. No positional parameters allowed. Unlike command-line options, " +
- "unrecognized options are ignored. " + "A single-valued option set in an options file may be overridden " +
- "by a subsequent command-line option. " +
- "A line starting with '#' is considered a comment.", false, true, 0, Integer.MAX_VALUE, null, new String[0]);
- printOptionUsage(stream, optionsFileOptionDefinition);
- }
-
- /**
- * Parse command-line options, and store values in callerOptions object passed to ctor.
- * @param messageStream Where to write error messages.
- * @param args Command line tokens.
- * @return true if command line is valid.
- */
- public boolean parseOptions(final PrintStream messageStream, final String[] args) {
- this.argv = args;
- this.messageStream = messageStream;
- for (final String arg: args) {
- if (arg.equals("-h") || arg.equals("--help")) {
- usage(messageStream);
- return false;
- }
- final String[] pair = arg.split("=", 2);
- if (pair.length == 2) {
- if (pair[0].equals(OPTIONS_FILE)) {
- if (!parseOptionsFile(pair[1])) {
- messageStream.println();
- usage(messageStream);
- return false;
- }
- } else {
- if (!parseOption(pair[0], pair[1], false)) {
- messageStream.println();
- usage(messageStream);
- return false;
- }
- }
- } else if (!parsePositionalArgument(arg)) {
- messageStream.println();
- usage(messageStream);
- return false;
- }
- }
- if (!checkNumArguments()) {
- messageStream.println();
- usage(messageStream);
- return false;
- }
- return true;
- }
-
- /**
- * After command line has been parsed, make sure that all required options have values, and that
- * lists with minimum # of elements have sufficient.
- * @return true if valid
- */
- private boolean checkNumArguments() {
- try {
- for (final OptionDefinition optionDefinition : optionDefinitions) {
- StringBuilder mutextOptionNames = new StringBuilder();
- for (String mutexOption : optionDefinition.mutuallyExclusive) {
- OptionDefinition mutextOptionDef = optionMap.get(mutexOption);
- if (mutextOptionDef != null && mutextOptionDef.hasBeenSet) {
- mutextOptionNames.append(" ").append(mutextOptionDef.name);
- }
- }
- if (optionDefinition.hasBeenSet && mutextOptionNames.length() > 0) {
- messageStream.println("ERROR: Option '" + optionDefinition.name +
- "' cannot be used in conjunction with option(s)" +
- mutextOptionNames.toString());
- return false;
- }
- if (optionDefinition.isCollection) {
- final Collection c = (Collection)optionDefinition.field.get(callerOptions);
- if (c.size() < optionDefinition.minElements) {
- messageStream.println("ERROR: Option '" + optionDefinition.name + "' must be specified at least " +
- optionDefinition.minElements + " times.");
- return false;
- }
- } else if (!optionDefinition.optional && !optionDefinition.hasBeenSet && mutextOptionNames.length() == 0) {
- messageStream.print("ERROR: Option '" + optionDefinition.name + "' is required");
- if (optionDefinition.mutuallyExclusive.isEmpty()) {
- messageStream.println(".");
- } else {
- messageStream.println(" unless any of " + optionDefinition.mutuallyExclusive + " are specified.");
- }
- return false;
- }
- }
- if (positionalArguments != null) {
- final Collection c = (Collection)positionalArguments.get(callerOptions);
- if (c.size() < minPositionalArguments) {
- messageStream.println("ERROR: At least " + minPositionalArguments +
- " positional arguments must be specified.");
- return false;
- }
- }
- return true;
- } catch (IllegalAccessException e) {
- // Should never happen because lack of publicness has already been checked.
- throw new RuntimeException(e);
- }
- }
-
- private boolean parsePositionalArgument(final String stringValue) {
- if (positionalArguments == null) {
- messageStream.println("ERROR: Invalid argument '" + stringValue + "'.");
- return false;
- }
- final Object value;
- try {
- value = constructFromString(getUnderlyingType(positionalArguments), stringValue);
- } catch (CommandLineParseException e) {
- messageStream.println("ERROR: " + e.getMessage());
- return false;
- }
- final Collection c;
- try {
- c = (Collection)positionalArguments.get(callerOptions);
- } catch (IllegalAccessException e) {
- throw new RuntimeException(e);
- }
- if (c.size() >= maxPositionalArguments) {
- messageStream.println("ERROR: No more than " + maxPositionalArguments +
- " positional arguments may be specified on the command line.");
- return false;
- }
- c.add(value);
- return true;
- }
-
- private boolean parseOption(String key, final String stringValue, final boolean optionsFile) {
- key = key.toUpperCase();
- final OptionDefinition optionDefinition = optionMap.get(key);
- if (optionDefinition == null) {
- if (optionsFile) {
- // Silently ignore unrecognized option from options file
- return true;
- }
- messageStream.println("ERROR: Unrecognized option: " + key);
- return false;
- }
- if (!optionDefinition.isCollection) {
- if (optionDefinition.hasBeenSet && !optionDefinition.hasBeenSetFromOptionsFile) {
- messageStream.println("ERROR: Option '" + key + "' cannot be specified more than once.");
- return false;
- }
- }
- final Object value;
- try {
- value = constructFromString(getUnderlyingType(optionDefinition.field), stringValue);
- } catch (CommandLineParseException e) {
- messageStream.println("ERROR: " + e.getMessage());
- return false;
- }
- try {
- if (optionDefinition.isCollection) {
- final Collection c = (Collection)optionDefinition.field.get(callerOptions);
- if (c.size() >= optionDefinition.maxElements) {
- messageStream.println("ERROR: Option '" + key + "' cannot be used more than " +
- optionDefinition.maxElements + " times.");
- return false;
- }
- c.add(value);
- } else {
- optionDefinition.field.set(callerOptions, value);
- optionDefinition.hasBeenSet = true;
- optionDefinition.hasBeenSetFromOptionsFile = optionsFile;
- }
- } catch (IllegalAccessException e) {
- // Should never happen because we only iterate through public fields.
- throw new RuntimeException(e);
- }
- return true;
- }
-
- /**
- * Parsing of options from file is looser than normal. Any unrecognized options are
- * ignored, and a single-valued option that is set in a file may be overridden by a
- * subsequent appearance of that option.
- * A line that starts with '#' is ignored.
- * @param optionsFile
- * @return false if a fatal error occurred
- */
- private boolean parseOptionsFile(final String optionsFile) {
- try {
- final BufferedReader reader = new BufferedReader(new FileReader(optionsFile));
- String line;
- while ((line = reader.readLine()) != null) {
- if (line.startsWith("#")) {
- continue;
- }
- final String[] pair = line.split("=", 2);
- if (pair.length == 2) {
- if (!parseOption(pair[0], pair[1], true)) {
- messageStream.println();
- usage(messageStream);
- return false;
- }
- } else {
- messageStream.println("Strange line in OPTIONS_FILE " + optionsFile + ": " + line);
- usage(messageStream);
- return false;
- }
- }
- reader.close();
- return true;
-
- } catch (IOException e) {
- throw new PicardException("I/O error loading OPTIONS_FILE=" + optionsFile, e);
- }
- }
-
- private void printOptionUsage(final PrintStream stream, final OptionDefinition optionDefinition) {
- final String type = getUnderlyingType(optionDefinition.field).getSimpleName();
- String optionLabel = optionDefinition.name + "=" + type;
- stream.print(optionLabel);
- if (optionDefinition.shortName.length() > 0) {
- stream.println();
- }
- if (optionDefinition.shortName.length() > 0) {
- optionLabel = optionDefinition.shortName + "=" + type;
- stream.print(optionLabel);
- }
- int numSpaces = OPTION_COLUMN_WIDTH - optionLabel.length();
- if (optionLabel.length() > OPTION_COLUMN_WIDTH) {
- stream.println();
- numSpaces = OPTION_COLUMN_WIDTH;
- }
- printSpaces(stream, numSpaces);
- final StringBuilder sb = new StringBuilder();
- if (optionDefinition.doc.length() > 0) {
- sb.append(optionDefinition.doc);
- sb.append(" ");
- }
- if (optionDefinition.optional && !optionDefinition.isCollection) {
- sb.append("Default value: ");
- sb.append(optionDefinition.defaultValue);
- sb.append(". ");
- } else if (!optionDefinition.isCollection){
- sb.append("Required. ");
- }
- Object[] enumConstants = getUnderlyingType(optionDefinition.field).getEnumConstants();
- if (enumConstants == null && getUnderlyingType(optionDefinition.field) == Boolean.class) {
- enumConstants = TRUE_FALSE_VALUES;
- }
- if (enumConstants != null) {
- sb.append("Possible values: {");
- for (int i = 0; i < enumConstants.length; ++i) {
- if (i > 0) {
- sb.append(", ");
- }
- sb.append(enumConstants[i].toString());
- }
- sb.append("} ");
- }
- if (optionDefinition.isCollection) {
- if (optionDefinition.minElements == 0) {
- if (optionDefinition.maxElements == Integer.MAX_VALUE) {
- sb.append("This option may be specified 0 or more times.");
- } else {
- sb.append("This option must be specified no more than " + optionDefinition.maxElements + "times.");
- }
- } else if (optionDefinition.maxElements == Integer.MAX_VALUE) {
- sb.append("This option must be specified at least " + optionDefinition.minElements + " times.");
- } else {
- sb.append("This option may be specified between " + optionDefinition.minElements +
- " and " + optionDefinition.maxElements + " times.");
- }
- }
- if (!optionDefinition.mutuallyExclusive.isEmpty()) {
- sb.append(" Cannot be used in conjuction with option(s)");
- for (String option : optionDefinition.mutuallyExclusive) {
- OptionDefinition mutextOptionDefinition = optionMap.get(option);
- sb.append(" ").append(mutextOptionDefinition.name);
- if (mutextOptionDefinition.shortName.length() > 0) {
- sb.append(" (").append(mutextOptionDefinition.shortName).append(")");
- }
- }
- }
- final String wrappedDescription = StringUtil.wordWrap(sb.toString(), DESCRIPTION_COLUMN_WIDTH);
- final String[] descriptionLines = wrappedDescription.split("\n");
- for (int i = 0; i < descriptionLines.length; ++i) {
- if (i > 0) {
- printSpaces(stream, OPTION_COLUMN_WIDTH);
- }
- stream.println(descriptionLines[i]);
- }
- stream.println();
- }
-
- private void printSpaces(final PrintStream stream, final int numSpaces) {
- final StringBuilder sb = new StringBuilder();
- for (int i = 0; i < numSpaces; ++i) {
- sb.append(" ");
- }
- stream.print(sb);
- }
-
- private void handleOptionAnnotation(final Field field) {
- try {
- final Option optionAnnotation = field.getAnnotation(Option.class);
- final boolean isCollection = isCollectionField(field);
- if (isCollection) {
- if (optionAnnotation.maxElements() == 0) {
- throw new CommandLineParserDefinitionException("@Option member " + field.getName() +
- "has maxElements = 0");
- }
- if (optionAnnotation.minElements() > optionAnnotation.maxElements()) {
- throw new CommandLineParserDefinitionException("In @Option member " + field.getName() +
- ", minElements cannot be > maxElements");
- }
- }
- if (!canBeMadeFromString(getUnderlyingType(field))) {
- throw new CommandLineParserDefinitionException("@Option member " + field.getName() +
- " must have a String ctor or be an enum");
- }
-
- final OptionDefinition optionDefinition = new OptionDefinition(field,
- field.getName(),
- optionAnnotation.shortName(),
- optionAnnotation.doc(), optionAnnotation.optional() || (field.get(callerOptions) != null),
- isCollection, optionAnnotation.minElements(),
- optionAnnotation.maxElements(), field.get(callerOptions),
- optionAnnotation.mutex());
-
- for (String option : optionAnnotation.mutex()) {
- OptionDefinition mutextOptionDef = optionMap.get(option);
- if (mutextOptionDef != null) {
- mutextOptionDef.mutuallyExclusive.add(field.getName());
- }
- }
- if (optionMap.containsKey(optionDefinition.name)) {
- throw new CommandLineParserDefinitionException(optionDefinition.name + " has already been used");
- }
- optionMap.put(optionDefinition.name, optionDefinition);
- if (optionDefinition.shortName.length() > 0) {
- if (optionMap.containsKey(optionDefinition.shortName)) {
- throw new CommandLineParserDefinitionException(optionDefinition.shortName + " has already been used");
- }
- optionMap.put(optionDefinition.shortName, optionDefinition);
- }
- optionDefinitions.add(optionDefinition);
- } catch (IllegalAccessException e) {
- throw new CommandLineParserDefinitionException(field.getName() +
- " must have public visibility to have @Option annotation");
- }
- }
-
- private void handleUsageAnnotation(final Field field) {
- if (usagePreamble != null) {
- throw new CommandLineParserDefinitionException
- ("@Usage cannot be used more than once in an option class.");
- }
- try {
- usagePreamble = (String)field.get(callerOptions);
- final Usage usageAnnotation = field.getAnnotation(Usage.class);
- if (usageAnnotation.programVersion().length() > 0) {
- usagePreamble += "Version: " + usageAnnotation.programVersion() + "\n";
- }
- } catch (IllegalAccessException e) {
- throw new CommandLineParserDefinitionException("@Usage data member must be public");
- } catch (ClassCastException e) {
- throw new CommandLineParserDefinitionException
- ("@Usage can only be applied to a String data member.");
- }
- }
-
- private void handlePositionalArgumentAnnotation(final Field field) {
- if (positionalArguments != null) {
- throw new CommandLineParserDefinitionException
- ("@PositionalArguments cannot be used more than once in an option class.");
- }
- positionalArguments = field;
- if (!isCollectionField(field)) {
- throw new CommandLineParserDefinitionException("@PositionalArguments must be applied to a Collection");
- }
-
- if (!canBeMadeFromString(getUnderlyingType(field))) {
- throw new CommandLineParserDefinitionException("@PositionalParameters member " + field.getName() +
- "does not have a String ctor");
- }
-
- final PositionalArguments positionalArgumentsAnnotation = field.getAnnotation(PositionalArguments.class);
- minPositionalArguments = positionalArgumentsAnnotation.minElements();
- maxPositionalArguments = positionalArgumentsAnnotation.maxElements();
- if (minPositionalArguments > maxPositionalArguments) {
- throw new CommandLineParserDefinitionException("In @PositionalArguments, minElements cannot be > maxElements");
- }
- }
-
- private boolean isCollectionField(final Field field) {
- try {
- field.getType().asSubclass(Collection.class);
- return true;
- } catch (ClassCastException e) {
- return false;
- }
- }
-
- private Class getUnderlyingType(final Field field) {
- if (isCollectionField(field)) {
- final ParameterizedType clazz = (ParameterizedType)(field.getGenericType());
- final Type[] genericTypes = clazz.getActualTypeArguments();
- if (genericTypes.length != 1) {
- throw new CommandLineParserDefinitionException("Strange collection type for field " + field.getName());
- }
- return (Class)genericTypes[0];
-
- } else {
- return field.getType();
- }
- }
-
- // True if clazz is an enum, or if it has a ctor that takes a single String argument.
- private boolean canBeMadeFromString(final Class clazz) {
- if (clazz.isEnum()) {
- return true;
- }
- try {
- clazz.getConstructor(String.class);
- return true;
- } catch (NoSuchMethodException e) {
- return false;
- }
- }
-
- private Object constructFromString(final Class clazz, final String s) {
- try {
- if (clazz.isEnum()) {
- try {
- return Enum.valueOf(clazz, s);
- } catch (IllegalArgumentException e) {
- throw new CommandLineParseException("'" + s + "' is not a valid value for " +
- clazz.getSimpleName() + ".", e);
- }
- }
- final Constructor ctor = clazz.getConstructor(String.class);
- return ctor.newInstance(s);
- } catch (NoSuchMethodException e) {
- // Shouldn't happen because we've checked for presence of ctor
- throw new CommandLineParseException(e);
- } catch (InstantiationException e) {
- throw new CommandLineParseException("Abstract class '" + clazz.getSimpleName() +
- "'cannot be used for an option value type.", e);
- } catch (IllegalAccessException e) {
- throw new CommandLineParseException("String constructor for option value type '" + clazz.getSimpleName() +
- "' must be public.", e);
- } catch (InvocationTargetException e) {
- throw new CommandLineParseException("Problem constructing " + clazz.getSimpleName() + " from the string '" + s + "'.",
- e.getCause());
- }
- }
-
- public String[] getArgv() {
- return argv;
- }
-
- private class OptionDefinition {
- final Field field;
- final String name;
- final String shortName;
- final String doc;
- final boolean optional;
- final boolean isCollection;
- final int minElements;
- final int maxElements;
- final String defaultValue;
- boolean hasBeenSet = false;
- boolean hasBeenSetFromOptionsFile = false;
- Set mutuallyExclusive;
-
- private OptionDefinition(final Field field, final String name, final String shortName, final String doc, final boolean optional, final boolean collection,
- final int minElements, final int maxElements, final Object defaultValue, String[] mutuallyExclusive) {
- this.field = field;
- this.name = name.toUpperCase();
- this.shortName = shortName.toUpperCase();
- this.doc = doc;
- this.optional = optional;
- isCollection = collection;
- this.minElements = minElements;
- this.maxElements = maxElements;
- if (defaultValue != null) {
- this.defaultValue = defaultValue.toString();
- } else {
- this.defaultValue = "null";
- }
- this.mutuallyExclusive = new HashSet(Arrays.asList(mutuallyExclusive));
- }
- }
-}
diff --git a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParserDefinitionException.java b/java/lib/edu/mit/broad/picard/cmdline/CommandLineParserDefinitionException.java
deleted file mode 100644
index 088755e2a..000000000
--- a/java/lib/edu/mit/broad/picard/cmdline/CommandLineParserDefinitionException.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2008 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.cmdline;
-
-public class CommandLineParserDefinitionException extends RuntimeException {
- public CommandLineParserDefinitionException() {
- }
-
- public CommandLineParserDefinitionException(String s) {
- super(s);
- }
-
- public CommandLineParserDefinitionException(String s, Throwable throwable) {
- super(s, throwable);
- }
-
- public CommandLineParserDefinitionException(Throwable throwable) {
- super(throwable);
- }
-}
diff --git a/java/lib/edu/mit/broad/picard/cmdline/CommandLineProgram.java b/java/lib/edu/mit/broad/picard/cmdline/CommandLineProgram.java
deleted file mode 100644
index 10ee7635f..000000000
--- a/java/lib/edu/mit/broad/picard/cmdline/CommandLineProgram.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-* The Broad Institute
-* SOFTWARE COPYRIGHT NOTICE AGREEMENT
-* This software and its documentation are copyright 2009 by the
-* Broad Institute/Massachusetts Institute of Technology. All rights are reserved.
-*
-* This software is supplied without any warranty or guaranteed support whatsoever. Neither
-* the Broad Institute nor MIT can be responsible for its use, misuse, or functionality.
-*/
-package edu.mit.broad.picard.cmdline;
-
-import edu.mit.broad.picard.util.Log;
-import edu.mit.broad.picard.util.StringUtil;
-import edu.mit.broad.picard.metrics.Header;
-import edu.mit.broad.picard.metrics.StringHeader;
-import edu.mit.broad.picard.metrics.MetricsFile;
-import edu.mit.broad.picard.metrics.MetricBase;
-
-import java.io.File;
-import java.util.Date;
-import java.util.List;
-import java.util.ArrayList;
-
-/**
- * Abstract class to facilitate writing command-line programs.
- *
- * To use:
- *
- * 1. Extend this class with a concrete class that has data members annotated with @Option, @PositionalArguments
- * and/or @Usage annotations.
- *
- * 2. If there is any custom command-line validation, override customCommandLineValidation(). When this method is
- * called, the command line has been parsed and set into the data members of the concrete class.
- *
- * 3. Implement a method doWork(). This is called after successful comand-line processing. The value it returns is
- * the exit status of the program. It is assumed that the concrete class emits any appropriate error message before
- * returning non-zero. doWork() may throw unchecked exceptions, which are caught and reported appropriately.
- *
- * 4. Implement the following static method in the concrete class:
- *
- * public static void main(String[] argv) {
- System.exit(new MyConcreteClass().instanceMain(argv));
- }
-
-
- */
-public abstract class CommandLineProgram {
-
- @Option
- public File TMP_DIR = new File(System.getProperty("java.io.tmpdir"), System.getProperty("user.name"));
-
- @Option(doc = "Control verbosity of logging")
- public Log.LogLevel VERBOSITY = Log.LogLevel.INFO;
-
- @Option(doc = "Whether to suppress job-summary info on System.out")
- public Boolean QUIET = false;
-
- private final String standardUsagePreamble = CommandLineParser.getStandardUsagePreamble(getClass());
-
- /**
- * Initialized in parseArgs. Subclasses may want to access this to do
- * their own validation, and then print usage using clp.
- */
- protected CommandLineParser clp;
-
- private final List defaultHeaders = new ArrayList