Update GATK to work with latest htsjdk

ValidationStringency was moved from htsjdk.samtools.SAMFileReader to htsjdk.samtools
samtools find BAM index file method was also moved (and made public!)

public/gatk-queue-extensions-public/src/main/scala/org/broadinstitute/gatk/queue/extensions/picard/PicardBamFunction.scala

@@ -27,7 +27,7 @@ package org.broadinstitute.gatk.queue.extensions.picard
 
 import java.io.File
 import org.broadinstitute.gatk.queue.function.JavaCommandLineFunction
-import htsjdk.samtools.SAMFileReader.ValidationStringency
+import htsjdk.samtools.ValidationStringency
 import htsjdk.samtools.SAMFileHeader.SortOrder
 
 /**

public/gatk-queue-extensions-public/src/main/scala/org/broadinstitute/gatk/queue/extensions/picard/PicardMetricsFunction.scala

@@ -27,7 +27,7 @@ package org.broadinstitute.gatk.queue.extensions.picard
 
 import java.io.File
 import org.broadinstitute.gatk.queue.function.JavaCommandLineFunction
-import htsjdk.samtools.SAMFileReader.ValidationStringency
+import htsjdk.samtools.ValidationStringency
 import htsjdk.samtools.SAMFileHeader.SortOrder
 
 /**

public/gatk-root/pom.xml

@@ -44,8 +44,8 @@
         <test.listeners>org.testng.reporters.FailedReporter,org.testng.reporters.JUnitXMLReporter,org.broadinstitute.gatk.utils.TestNGTestTransformer,org.broadinstitute.gatk.utils.GATKTextReporter,org.uncommons.reportng.HTMLReporter</test.listeners>
 
         <!-- Version numbers for picard and htsjdk -->
-        <htsjdk.version>1.112.1452</htsjdk.version>
+        <htsjdk.version>1.118.1556</htsjdk.version>
-        <picard.version>1.112.1452</picard.version>
+        <picard.version>1.118.1521</picard.version>
     </properties>
 
     <!-- Dependency configuration (versions, etc.) -->

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/ReadProperties.java

@@ -27,6 +27,7 @@ package org.broadinstitute.gatk.engine;
 
 import htsjdk.samtools.SAMFileHeader;
 import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.ValidationStringency;
 import org.broadinstitute.gatk.engine.arguments.ValidationExclusion;
 import org.broadinstitute.gatk.engine.datasources.reads.SAMReaderID;
 import org.broadinstitute.gatk.engine.downsampling.DownsamplingMethod;
@@ -56,7 +57,7 @@ public class ReadProperties {
     private final Collection<SAMReaderID> readers;
     private final SAMFileHeader header;
     private final SAMFileHeader.SortOrder sortOrder;
-    private final SAMFileReader.ValidationStringency validationStringency;
+    private final ValidationStringency validationStringency;
     private final DownsamplingMethod downsamplingMethod;
     private final ValidationExclusion exclusionList;
     private final Collection<ReadFilter> supplementalFilters;
@@ -107,7 +108,7 @@ public class ReadProperties {
      * How strict should validation be?
      * @return Stringency of validation.
      */
-    public SAMFileReader.ValidationStringency getValidationStringency() {
+    public ValidationStringency getValidationStringency() {
         return validationStringency;
     }
 
@@ -173,7 +174,7 @@ public class ReadProperties {
            SAMFileHeader header,
            SAMFileHeader.SortOrder sortOrder,
            boolean useOriginalBaseQualities,
-           SAMFileReader.ValidationStringency strictness,
+           ValidationStringency strictness,
            DownsamplingMethod downsamplingMethod,
            ValidationExclusion exclusionList,
            Collection<ReadFilter> supplementalFilters,

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/alignment/bwa/java/AlignerTestHarness.java

@@ -63,7 +63,7 @@ public class AlignerTestHarness {
         int count = 0;
 
         SAMFileReader reader = new SAMFileReader(bamFile);
-        reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
+        reader.setValidationStringency(ValidationStringency.SILENT);
 
         int mismatches = 0;
         int failures = 0;

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/arguments/GATKArgumentCollection.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.gatk.engine.arguments;
 
 import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.ValidationStringency;
 import org.broadinstitute.gatk.utils.commandline.*;
 import org.broadinstitute.gatk.engine.GenomeAnalysisEngine;
 import org.broadinstitute.gatk.engine.downsampling.DownsampleType;
@@ -361,7 +362,7 @@ public class GATKArgumentCollection {
      * Keep in mind that if you set this to LENIENT, we may refuse to provide you with support if anything goes wrong.
      */
     @Argument(fullName = "validation_strictness", shortName = "S", doc = "How strict should we be with validation", required = false)
-    public SAMFileReader.ValidationStringency strictnessLevel = SAMFileReader.ValidationStringency.SILENT;
+    public ValidationStringency strictnessLevel = ValidationStringency.SILENT;
     /**
      * Some tools keep program records in the SAM header by default. Use this argument to override that behavior and discard program records for the SAM header.
      */

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/SAMDataSource.java

@@ -86,7 +86,7 @@ public class SAMDataSource {
     /**
      * How strict are the readers driving this data source.
      */
-    private final SAMFileReader.ValidationStringency validationStringency;
+    private final ValidationStringency validationStringency;
 
     /**
      * Do we want to remove the program records from this data source?
@@ -183,7 +183,7 @@ public class SAMDataSource {
                 numFileHandles,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -202,7 +202,7 @@ public class SAMDataSource {
             Integer numFileHandles,
             GenomeLocParser genomeLocParser,
             boolean useOriginalBaseQualities,
-            SAMFileReader.ValidationStringency strictness,
+            ValidationStringency strictness,
             Integer readBufferSize,
             DownsamplingMethod downsamplingMethod,
             ValidationExclusion exclusionList,
@@ -251,7 +251,7 @@ public class SAMDataSource {
             Integer numFileHandles,
             GenomeLocParser genomeLocParser,
             boolean useOriginalBaseQualities,
-            SAMFileReader.ValidationStringency strictness,
+            ValidationStringency strictness,
             Integer readBufferSize,
             DownsamplingMethod downsamplingMethod,
             ValidationExclusion exclusionList,
@@ -800,7 +800,7 @@ public class SAMDataSource {
          * @param reader Reader for which to determine the id.
          * @return id of the given reader.
          */
-        protected synchronized SAMReaderID getReaderID(SAMFileReader reader) {
+        protected synchronized SAMReaderID getReaderID(SamReader reader) {
             for(SAMReaders readers: allResources) {
                 SAMReaderID id = readers.getReaderID(reader);
                 if(id != null)
@@ -845,7 +845,7 @@ public class SAMDataSource {
          * @param validationStringency validation stringency.
          * @param removeProgramRecords indicate whether to clear program records from the readers
          */
-        public SAMReaders(Collection<SAMReaderID> readerIDs, SAMFileReader.ValidationStringency validationStringency, boolean removeProgramRecords) {
+        public SAMReaders(Collection<SAMReaderID> readerIDs, ValidationStringency validationStringency, boolean removeProgramRecords) {
             final int totalNumberOfFiles = readerIDs.size();
             int readerNumber = 1;
             final SimpleTimer timer = new SimpleTimer().start();
@@ -1028,7 +1028,7 @@ public class SAMDataSource {
          * @param reader Reader for which to search.
          * @return The id associated the given reader, or null if the reader is not present in this collection.
          */
-        protected SAMReaderID getReaderID(SAMFileReader reader) {
+        protected SAMReaderID getReaderID(SamReader reader) {
             for(Map.Entry<SAMReaderID,SAMFileReader> entry: readers.entrySet()) {
                 if(reader == entry.getValue())
                     return entry.getKey();
@@ -1132,33 +1132,11 @@ public class SAMDataSource {
 
     /**
      * Locates the index file alongside the given BAM, if present.
-     * TODO: This is currently a hachetjob that reaches into Picard and pulls out its index file locator.  Replace with something more permanent.
      * @param bamFile The data file to use.
      * @return A File object if the index file is present; null otherwise.
      */
     private File findIndexFile(File bamFile) {
-        File indexFile;
+        return SamFiles.findIndex(bamFile);
-
-        try {
-            Class bamFileReaderClass = Class.forName("htsjdk.samtools.BAMFileReader");
-            Method indexFileLocator = bamFileReaderClass.getDeclaredMethod("findIndexFile",File.class);
-            indexFileLocator.setAccessible(true);
-            indexFile = (File)indexFileLocator.invoke(null,bamFile);
-        }
-        catch(ClassNotFoundException ex) {
-            throw new ReviewedGATKException("Unable to locate BAMFileReader class, used to check for index files");
-        }
-        catch(NoSuchMethodException ex) {
-            throw new ReviewedGATKException("Unable to locate Picard index file locator.");
-        }
-        catch(IllegalAccessException ex) {
-            throw new ReviewedGATKException("Unable to access Picard index file locator.");
-        }
-        catch(InvocationTargetException ex) {
-            throw new ReviewedGATKException("Unable to invoke Picard index file locator.");
-        }
-
-        return indexFile;
     }
 
     /**

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/BAMFileStat.java

@@ -27,6 +27,7 @@ package org.broadinstitute.gatk.engine.datasources.reads.utilities;
 
 import htsjdk.samtools.BAMIndex;
 import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.ValidationStringency;
 import org.broadinstitute.gatk.utils.commandline.Argument;
 import org.broadinstitute.gatk.utils.commandline.CommandLineProgram;
 import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException;
@@ -85,7 +86,7 @@ public class BAMFileStat extends CommandLineProgram {
         BAMIndex index;
 
         reader = new SAMFileReader(bamFile);
-        reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
+        reader.setValidationStringency(ValidationStringency.SILENT);
         reader.enableIndexCaching(true);
         index = reader.getIndex();
 

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/datasources/reads/utilities/PrintBAMRegion.java

@@ -25,10 +25,7 @@
 
 package org.broadinstitute.gatk.engine.datasources.reads.utilities;
 
-import htsjdk.samtools.GATKBAMFileSpan;
+import htsjdk.samtools.*;
-import htsjdk.samtools.GATKChunk;
-import htsjdk.samtools.SAMFileReader;
-import htsjdk.samtools.SAMRecordIterator;
 import org.broadinstitute.gatk.utils.commandline.Argument;
 import org.broadinstitute.gatk.utils.commandline.CommandLineProgram;
 import org.broadinstitute.gatk.utils.exceptions.UserException;
@@ -58,7 +55,7 @@ public class PrintBAMRegion extends CommandLineProgram {
 
     public int execute() {
         SAMFileReader reader = new SAMFileReader(input);
-        reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
+        reader.setValidationStringency(ValidationStringency.SILENT);
 
         Pattern regionPattern = Pattern.compile("(\\d+):(\\d+)-(\\d+):(\\d+)");
         Matcher matcher = regionPattern.matcher(region);

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/io/OutputTracker.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.gatk.engine.io;
 
 import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.ValidationStringency;
 import org.broadinstitute.gatk.utils.commandline.ArgumentSource;
 import org.broadinstitute.gatk.engine.io.storage.Storage;
 import org.broadinstitute.gatk.engine.io.storage.StorageFactory;
@@ -77,7 +78,7 @@ public abstract class OutputTracker {
      */
     public abstract <T> T getStorage( Stub<T> stub );
 
-    public void prepareWalker( Walker walker, SAMFileReader.ValidationStringency strictnessLevel ) {
+    public void prepareWalker( Walker walker, ValidationStringency strictnessLevel ) {
         for( Map.Entry<ArgumentSource,Object> io: inputs.entrySet() ) {
             ArgumentSource targetField = io.getKey();
             Object targetValue = io.getValue();

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/engine/walkers/diffengine/BAMDiffableReader.java

@@ -28,6 +28,7 @@ package org.broadinstitute.gatk.engine.walkers.diffengine;
 import htsjdk.samtools.SAMFileReader;
 import htsjdk.samtools.SAMRecord;
 import htsjdk.samtools.SAMRecordIterator;
+import htsjdk.samtools.ValidationStringency;
 import htsjdk.samtools.util.BlockCompressedInputStream;
 
 import java.io.*;
@@ -49,7 +50,7 @@ public class BAMDiffableReader implements DiffableReader {
     @Override
     public DiffElement readFromFile(File file, int maxElementsToRead) {
         final SAMFileReader reader = new SAMFileReader(file, null); // null because we don't want it to look for the index
-        reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
+        reader.setValidationStringency(ValidationStringency.SILENT);
 
         DiffNode root = DiffNode.rooted(file.getName());
         SAMRecordIterator iterator = reader.iterator();

public/gatk-tools-public/src/main/java/org/broadinstitute/gatk/utils/sam/SAMFileReaderBuilder.java

@@ -26,6 +26,7 @@
 package org.broadinstitute.gatk.utils.sam;
 
 import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.ValidationStringency;
 import org.broadinstitute.gatk.utils.exceptions.ReviewedGATKException;
 
 import java.io.File;
@@ -47,7 +48,7 @@ public class SAMFileReaderBuilder {
     /**
      * What compression level should be used when building this file?
      */
-    private SAMFileReader.ValidationStringency validationStringency = null;
+    private ValidationStringency validationStringency = null;
 
     /**
      * Sets the handle of the sam file to which data should be written.
@@ -61,7 +62,7 @@ public class SAMFileReaderBuilder {
      * Sets the validation stringency to apply when reading this sam file.
      * @param validationStringency Stringency to apply.  Must not be null.
      */
-    public void setValidationStringency( SAMFileReader.ValidationStringency validationStringency ) {
+    public void setValidationStringency( ValidationStringency validationStringency ) {
         this.validationStringency = validationStringency;
     }
 

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/EngineFeaturesIntegrationTest.java

@@ -25,10 +25,7 @@
 
 package org.broadinstitute.gatk.engine;
 
-import htsjdk.samtools.SAMFileReader;
+import htsjdk.samtools.*;
-import htsjdk.samtools.SAMReadGroupRecord;
-import htsjdk.samtools.SAMRecord;
-import htsjdk.samtools.SAMSequenceDictionary;
 import htsjdk.samtools.util.CloseableIterator;
 import htsjdk.tribble.readers.LineIterator;
 import org.broadinstitute.gatk.engine.walkers.WalkerTest;
@@ -308,7 +305,7 @@ public class EngineFeaturesIntegrationTest extends WalkerTest {
 
         final File outputBam = executeTest("testGATKEngineConsolidatesCigars", spec).first.get(0);
         final SAMFileReader reader = new SAMFileReader(outputBam);
-        reader.setValidationStringency(SAMFileReader.ValidationStringency.SILENT);
+        reader.setValidationStringency(ValidationStringency.SILENT);
         reader.setSAMRecordFactory(new GATKSamRecordFactory());
 
         final SAMRecord read = reader.iterator().next();

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/ReadMetricsUnitTest.java

@@ -152,7 +152,7 @@ public class ReadMetricsUnitTest extends BaseTest {
 
         final SAMDataSource dataSource = new SAMDataSource(samFiles, new ThreadAllocation(), null, genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -187,7 +187,7 @@ public class ReadMetricsUnitTest extends BaseTest {
 
         final SAMDataSource dataSource = new SAMDataSource(samFiles, new ThreadAllocation(), null, genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -228,7 +228,7 @@ public class ReadMetricsUnitTest extends BaseTest {
 
         final SAMDataSource dataSource = new SAMDataSource(samFiles, new ThreadAllocation(), null, genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -275,7 +275,7 @@ public class ReadMetricsUnitTest extends BaseTest {
 
         final SAMDataSource dataSource = new SAMDataSource(samFiles, new ThreadAllocation(), null, genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/datasources/reads/ReadShardBalancerUnitTest.java

@@ -88,7 +88,7 @@ public class ReadShardBalancerUnitTest extends BaseTest {
                                                          null,
                                                          new GenomeLocParser(header.getSequenceDictionary()),
                                                          false,
-                                                         SAMFileReader.ValidationStringency.SILENT,
+                                                         ValidationStringency.SILENT,
                                                          ReadShard.DEFAULT_MAX_READS,  // reset ReadShard.MAX_READS to ReadShard.DEFAULT_MAX_READS for each test
                                                          downsamplingMethod,
                                                          new ValidationExclusion(),

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/datasources/reads/SAMDataSourceUnitTest.java

@@ -105,7 +105,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -159,7 +159,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -175,7 +175,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -196,7 +196,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -221,7 +221,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),
@@ -238,7 +238,7 @@ public class SAMDataSourceUnitTest extends BaseTest {
                 null,
                 genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.SILENT,
+                ValidationStringency.SILENT,
                 null,
                 null,
                 new ValidationExclusion(),

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/filters/ReadFilterTest.java

@@ -317,7 +317,7 @@ public class ReadFilterTest extends BaseTest {
         final ThreadAllocation ta = new ThreadAllocation();
         final Integer numFileHandles = 1; // I believe that any value would do but need to confirm.
         final boolean useOriginalBaseQualities = true;
-        final SAMFileReader.ValidationStringency strictness = SAMFileReader.ValidationStringency.LENIENT;
+        final ValidationStringency strictness = ValidationStringency.LENIENT;
         final Integer readBufferSize = 1; // not relevant.
         final DownsamplingMethod downsamplingMethod = DownsamplingMethod.NONE;
         final ValidationExclusion exclusionList = composeValidationExclusion();

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/engine/traversals/TraverseActiveRegionsUnitTest.java

@@ -473,7 +473,7 @@ public class TraverseActiveRegionsUnitTest extends BaseTest {
 
         SAMDataSource dataSource = new SAMDataSource(samFiles, new ThreadAllocation(), null, genomeLocParser,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 null,
                 null,
                 new ValidationExclusion(),

public/gatk-tools-public/src/test/java/org/broadinstitute/gatk/utils/locusiterator/LocusIteratorByStateBaseTest.java

@@ -75,7 +75,7 @@ public class LocusIteratorByStateBaseTest extends BaseTest {
                 new SAMFileHeader(),
                 SAMFileHeader.SortOrder.coordinate,
                 false,
-                SAMFileReader.ValidationStringency.STRICT,
+                ValidationStringency.STRICT,
                 downsamplingMethod,
                 new ValidationExclusion(),
                 Collections.<ReadFilter>emptyList(),

public/repo/picard/picard/1.118.1521/picard-1.118.1521.pom

@@ -3,13 +3,13 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>picard</groupId>
     <artifactId>picard</artifactId>
-    <version>1.112.1452</version>
+    <version>1.118.1521</version>
     <name>picard</name>
     <dependencies>
         <dependency>
             <groupId>samtools</groupId>
             <artifactId>htsjdk</artifactId>
-            <version>1.112.1452</version>
+            <version>1.118.1556</version>
         </dependency>
         <!-- TODO: Picard is using a custom zip with just ant's BZip2 classes. See also: http://www.kohsuke.org/bzip2 -->
         <dependency>

public/repo/samtools/htsjdk/1.118.1556/htsjdk-1.118.1556.pom

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>samtools</groupId>
     <artifactId>htsjdk</artifactId>
-    <version>1.112.1452</version>
+    <version>1.118.1556</version>
     <name>htsjdk</name>
     <dependencies>
         <dependency>