Scala walkers can now be build and run like any other walker in the GATK. Added the getUrlsForClasspath to PackageUtils, the Reflections package isn't getting the manifest files from jars in the classpath, and so we weren't seeing any walkers outside of the GenomeAnalysisTK.jar.

A couple of notes:
-Commented out BaseTransitionTableCalculator.scala because it's won't build; Chris could you fix this one (or kill it if it's not needed).
-Removed the PrintReadsScala walker; moved the code over to a ScalaCountLoci walker (which is what the code was really doing).
-Added configurations items to the ivy xml file.



git-svn-id: file:///humgen/gsa-scr1/gsa-engineering/svn_contents/trunk@1956 348d0f76-0448-11de-a6fe-93d51630548a

build.xml

@@ -40,21 +40,18 @@
     <property name="ivy.jar.file" value="ivy-${ivy.install.version}.jar"/>
     <property name="ivy.settings.dir" value="settings"/>
     <property file="${ivy.settings.dir}/ivysettings.properties"/>
-
+    <property name="ivy.conf" value="default"/>
     <target name="resolve" depends="init"
             description="locate and download library dependencies">
-        <!-- retrieve ivy if necessary -->
       <mkdir dir="${ivy.jar.dir}"/>
       <get src="http://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/${ivy.jar.file}"
            dest="${ivy.jar.dir}/${ivy.jar.file}"
            usetimestamp="true"/>
-        <!-- initialize and load ivy -->
       <taskdef resource="org/apache/ivy/ant/antlib.xml"
                uri="antlib:org.apache.ivy.ant"
                classpath="${ivy.jar.dir}/${ivy.jar.file}"/>
       <ivy:settings file="${ivy.settings.dir}/ivysettings.xml"/>
-        <!-- retrieve dependencies -->
+      <ivy:retrieve file="ivy.xml" conf="${ivy.conf}" />
-        <ivy:retrieve/>
     </target>
 
 
@@ -104,10 +101,7 @@
         </jar>
 
         <jar jarfile="${dist}/GenomeAnalysisTK.jar">
-            <fileset dir="build">
+            <fileset dir="build" includes="**/gatk/**/*.class"/>
-              <include name="**/gatk/**/*.class"/>
-              <include name="**/alignment/**/*.class"/>
-            </fileset>
             <manifest>
                 <attribute name="Main-Class" value="org.broadinstitute.sting.gatk.CommandLineGATK"/>
             </manifest>
@@ -134,6 +128,7 @@
         <pathconvert property="jar.classpath" pathsep=" ">
             <flattenmapper/>
             <fileset dir="${dist}" includes="*.jar"/>
+            <filelist files="GATKScala.jar"/>
         </pathconvert>
 
         <jar jarfile="${dist}/StingUtils.jar" update="true">
@@ -186,48 +181,38 @@
         </javac>
     </target>
 
+    <!-- new scala target -->
     
-   <property name="scala.home" value="scala/installation"/>
+    <target name="scala" description="build the scala directory">
-   <property
+      <antcall target="resolve">
-      name="build.dir"
+        <param name="ivy.conf" value="scala"/>
-      value="${base.dir}/build"
+      </antcall>
-       />
+        <antcall target="dist"/>    
-    <target name="init.scala">
-      <property
-        name="scala-library.jar"
-        value="${scala.home}/lib/scala-library.jar"
-         />
-      <path id="build.classpath">
-        <pathelement location="${scala-library.jar}"   />
-        <!--<pathelement location="${your.path}"   />-->
-        <pathelement location="${build.dir}"   />
-      </path>
-     <taskdef name="scalac" classname="scala.tools.ant.Scalac">
-        <classpath>
-          <pathelement location="${scala.home}/lib/scala-compiler.jar"   />
-          <pathelement location="${scala-library.jar}"   />
-        </classpath>
-    </taskdef>
-    </target>
-    
       <property name="scala.src" value="scala/src" />
       <property name="scala.classes" value="scala/classes" />
-    <target name="scala" depends="init.scala" description="build the scala code">
+      <pathconvert property="scalajar.classpath" pathsep=";">
-        <echo message="Sting: Compiling scala!"/>
+        <flattenmapper/>
-        <mkdir dir="${scala.classes}"/>
+        <fileset dir="${basedir}" includes="dist/*.jar"/>
+      </pathconvert>
       
-        <scalac srcdir="${scala.src}" destdir="${scala.classes}" force="changed">
+      <path id="scala.classpath">
+        <pathelement location="lib/scala-compiler-2.7.7.jar"/>
+        <pathelement location="lib/scala-library-2.7.7.jar"/>
+        <fileset dir="${dist}">
+          <patternset id="scalaStuff">
+            <include name="*.jar"/>
+        </patternset>
+        </fileset>
+      </path>
+      <taskdef resource="scala/tools/ant/antlib.xml">
+        <classpath refid="scala.classpath"/>
+      </taskdef>
+      <mkdir dir="${scala.classes}"/>
+      <echo>Building Scala...</echo>
+      <scalac srcdir="${scala.src}" destdir="${scala.classes}" classpathref="scala.classpath" force="changed">
         <include name="**/*.scala"/>
-            <classpath>
-                <path refid="runtime.dependencies" />
-                <pathelement location="build"/>
-                <pathelement location="lib/junit-4.4.jar"/>
-                <pathelement location="${scala.home}/lib/scala-compiler.jar"   />
-                <pathelement location="${scala-library.jar}"   />
-          </classpath>
       </scalac>
       
-        <delete file="${dist}/GATKScala.jar"/>
       <jar jarfile="${dist}/GATKScala.jar">
         <fileset dir="${scala.classes}">
           <include name="**/*.class"/>
@@ -235,6 +220,9 @@
       </jar>
     </target>
     
+
+
+
     <!-- TEST -->
     <target name="test" depends="test.compile,agent" description="Run unit tests">
         <mkdir dir="${report}"/>
@@ -265,10 +253,13 @@
         <mkdir dir="${report}"/>
         <echo message="Sting: Running integration test cases!"/>
         <junit printsummary="yes" showoutput="yes" clonevm="yes" haltonfailure="yes" failureProperty="test.failure">
+
+            <jvmarg value="-Xdebug"/>
+            <jvmarg value="-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8090"/>
             <formatter type="${testOutput}" usefile="${usefile}" />
             <classpath>
                 <path refid="runtime.dependencies"/>
-                <pathelement location="build"/>
+                <pathelement location="${dist}/GenomeAnalysisTK.jar"/>
                 <pathelement location="${test.classes}"/>
                 <pathelement location="lib/junit-4.4.jar"/>
             </classpath>

ivy.xml

@@ -1,21 +1,30 @@
+
 <ivy-module version="1.0">
   <info organisation="org.broadinstitute" module="Sting"/>
-  <dependencies>
+  <configurations>
-    <dependency org="net.sf" name="sam" rev="latest.integration" />
+    <conf name="default" description="the core dependencies for the GATK"/>
-    <dependency org="net.sf" name="picard" rev="latest.integration" />
+    <conf name="scala" extends="default" description="the dependencies for scala"/>
-    <dependency org="edu.mit.broad" name="picard-private" rev="latest.integration" />
+  </configurations>
-    <dependency org="junit" name="junit" rev="4.4" />
+  <dependencies defaultconf="default">
-    <dependency org="log4j" name="log4j" rev="1.2.15" />
+    <dependency org="net.sf" name="sam" rev="latest.integration" conf="default"/>
-    <dependency org="colt" name="colt" rev="1.2.0" />
+    <dependency org="net.sf" name="picard" rev="latest.integration" conf="default"/>
-    <dependency org="jboss" name="javassist" rev="3.7.ga" />
+    <dependency org="edu.mit.broad" name="picard-private" rev="latest.integration" conf="default"/>
-    <dependency org="org.simpleframework" name="simple-xml" rev="2.0.4" />
+    <dependency org="junit" name="junit" rev="4.4" conf="default"/>
-    <dependency org="org.apache.bcel" name="bcel" rev="5.2" />
+    <dependency org="log4j" name="log4j" rev="1.2.15" conf="default"/>
+    <dependency org="colt" name="colt" rev="1.2.0" conf="default"/>
+    <dependency org="jboss" name="javassist" rev="3.7.ga" conf="default"/>
+    <dependency org="org.simpleframework" name="simple-xml" rev="2.0.4" conf="default"/>
+    <dependency org="org.apache.bcel" name="bcel" rev="5.2" conf="default"/>
    
     <!-- Dependencies for reflections mvn repository -->
-    <dependency org="org.reflections" name="reflections" rev="0.9.2" />
+    <dependency org="org.reflections" name="reflections" rev="0.9.2" conf="default"/>
-    <dependency org="com.google.collections" name="google-collections" rev="0.9" />
+    <dependency org="com.google.collections" name="google-collections" rev="0.9" conf="default"/>
 
-    <!-- JGraph -->
+    <!-- Dependencies for the graph aligner -->  
-    <dependency org="org.jgrapht" name="jgrapht-jdk1.5" rev="0.7.3" />    
+      <dependency org="org.jgrapht" name="jgrapht-jdk1.5" rev="0.7.3" conf="default"/>
+     
+    <!-- Scala dependancies -->
+    <dependency org="org.scala-lang" name="scala-compiler" rev="2.7.7" conf="scala->default"/>
+    <dependency org="org.scala-lang" name="scala-library" rev="2.7.7" conf="scala->default"/>
   </dependencies>
 </ivy-module>

java/src/org/broadinstitute/sting/utils/PackageUtils.java

@@ -3,16 +3,30 @@ package org.broadinstitute.sting.utils;
 import org.reflections.Reflections;
 import org.reflections.scanners.SubTypesScanner;
 import org.reflections.util.AbstractConfiguration;
-import org.reflections.util.ClasspathHelper;
+import org.apache.log4j.Logger;
 
 import java.util.Set;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Collection;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.io.File;
+import java.io.IOException;
+
+import com.google.common.collect.Lists;
 
 /**
  * PackageUtils contains some useful methods for package introspection.
  */
 public class PackageUtils {
+    /**
+     * our log, which we want to capture anything from this class
+     */
+    private static Logger logger = Logger.getLogger(PackageUtils.class);
+
     /**
      * A reference into our introspection utility.
      */
@@ -22,7 +36,7 @@ public class PackageUtils {
         // Initialize general-purpose source tree reflector.
         reflections = new Reflections(new AbstractConfiguration() {
             {
-                setUrls(ClasspathHelper.getUrlsForCurrentClasspath());
+                setUrls(PackageUtils.getUrlsForClasspath(System.getProperty("java.class.path")));
                 setScanners(new SubTypesScanner());
             }
         });
@@ -31,7 +45,9 @@ public class PackageUtils {
     /**
      * Private constructor.  No instantiating this class!
      */
-    private PackageUtils() {}
+    private PackageUtils() {
+    }
+
     {
     }
 
@@ -52,4 +68,93 @@ public class PackageUtils {
 
         return concreteTypes;
     }
+
+    /**
+     * get a list of URL's, given the current classpath.  This function is a fix for the
+     * reflections package, which doesn't correctly expand the classpath from a jar (it doesn't
+     * get the manifest and parse out the embedded classpth).
+     * @param classpath the current classpath
+     * @return
+     */
+    public static Collection<URL> getUrlsForClasspath(String classpath) {
+        // if the classpath is null, we can't really do anything
+        if (classpath == null || classpath.equals(""))
+            throw new StingException("Classpath cannot be empty or null");
+        List<URL> urls = Lists.newArrayList();
+
+        // the current working directory
+        String baseDir = System.getProperty("user.dir");
+
+        // our collection of classpath's to expand
+        List<JarPath> javaClassPath = new ArrayList<JarPath>();
+
+        // the current classpath can be a list of path's seperated by a semicolon
+        String[] classPaths = classpath.split(File.pathSeparator);
+        for (String part : classPaths) {
+            extractJarClasspath(baseDir, javaClassPath, part);
+        }
+
+        // check to make sure each extracted path exists, if so add it to our list
+        if (javaClassPath.size() > 0)
+            for (JarPath path : javaClassPath) {
+                try {
+                    if (path.isValid())
+                        urls.add(path.toValidFile().toURL());
+                } catch (MalformedURLException e) {
+                    throw new StingException("could not create url from " + path, e);
+                }
+            }
+
+        return urls;
+    }
+
+    /**
+     * extract the classpath from a jar
+     *
+     * @param baseDir       the base
+     * @param javaClassPath the list of jar paths
+     * @param part          the current the subsection of the classpath we're processing
+     */
+    private static void extractJarClasspath(String baseDir, List<JarPath> javaClassPath, String part) {
+        try {
+            JarFile myJar = new JarFile(part);
+            Attributes.Name classPath = new Attributes.Name("Class-Path");
+            if (myJar.getManifest().getMainAttributes().containsKey(classPath)) {
+                for (String jar : myJar.getManifest().getMainAttributes().getValue(classPath).split(" "))
+                    javaClassPath.add(new JarPath(baseDir, new File(part).getParent(), jar));
+            }
+        } catch (IOException e) {
+            logger.warn("could not retreive manifest from " + part);
+        }
+    }
+
+    /**
+     * a simple helper class, to determine the absolute path to a jar file
+     */
+    static class JarPath {
+        private final String mPath;
+        private final String mFilename;
+        private final String mWorkingDir;
+
+        public JarPath(String workingDir, String path, String filename) {
+            this.mPath = path;
+            this.mFilename = filename;
+            mWorkingDir = workingDir;
+        }
+
+        public File toValidFile() {
+            if (new File(mFilename).exists())
+                return new File(mFilename);
+            if (new File(mPath + File.separator + mFilename).exists())
+                return new File(mPath + File.separator + mFilename);
+            if (new File(mWorkingDir + File.separator + mFilename).exists())
+                return new File(mWorkingDir + File.separator + mFilename);
+            return null;
+        }
+
+        public boolean isValid() {
+            return (toValidFile() != null) ? true : false;
+        }
+    }
+
 }

scala/src/BaseTransitionTableCalculator.scala

@@ -1,4 +1,4 @@
-package org.broadinstitute.sting.scala
+/**package org.broadinstitute.sting.scala
 
 import gatk.walkers.genotyper.{UnifiedGenotyper, GenotypeCall}
 import java.io.File
@@ -68,7 +68,7 @@ class TransitionTable() {
   }
 }
 
-class BaseTransitionTableCalculator extends LocusWalker[Unit,Int] {
+class BaseTransitionTableCalculator { // extends LocusWalker[Unit,Int] {
   private var MIN_MAPPING_QUALITY = 30
   private var MIN_BASE_QUALITY = 20
   private var MIN_LOD = 5
@@ -208,3 +208,4 @@ class BaseTransitionTableCalculator extends LocusWalker[Unit,Int] {
     print1("1-mismatch-rev", tableREV)
   }
 }
+*/

scala/src/ScalaCountLoci.scala

@@ -5,9 +5,8 @@ import org.broadinstitute.sting.gatk.refdata.RefMetaDataTracker
 import org.broadinstitute.sting.gatk.contexts.ReferenceContext
 import org.broadinstitute.sting.gatk.contexts.AlignmentContext
 
-class PrintReadsScala extends LocusWalker[Int,Int] {
+class ScalaCountLoci extends LocusWalker[Int,Int] {
     override def map(tracker: RefMetaDataTracker, ref: ReferenceContext, context: AlignmentContext): Int = {
-        //println(context.getReads().size())
         return 1
     }